"""Offline unit tests for run_cian_city_sweep (#860). Pure & fast: NO live network, NO DB. Mirrors test_n1_city_sweep / test_city_sweep. Asserts orchestration semantics only. Coverage: - Все фазы (SERP → save → detail → houses) вызываются в нужном порядке - CianCitySweepCounters корректно заполняются - Cooperative cancel прерывает sweep перед anchor'ом - enrich_houses=False пропускает houses-фазу - Consecutive SERP failures → mark_banned + abort - Per-item detail/house error не валит весь sweep - mark_done вызывается при нормальном завершении """ from __future__ import annotations import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from typing import Any from unittest.mock import AsyncMock, MagicMock import pytest from app.services import scrape_pipeline from app.services.scrapers.base import ScrapedLot from app.services.scrapers.cian import CianScraper # Три тестовых anchor'а — не все 5, чтобы тесты были быстрее TEST_ANCHORS: list[tuple[float, float, str]] = [ (56.8400, 60.6050, "A1"), (56.7950, 60.5300, "A2"), (56.8970, 60.6100, "A3"), ] def _fake_lot(offer_id: str, *, nb: bool = False) -> ScrapedLot: """Создаём ScrapedLot с опциональным newbuilding-ссылкой.""" return ScrapedLot( source="cian", source_url=f"https://ekb.cian.ru/sale/flat/{offer_id}/", source_id=offer_id, address="Екатеринбург, улица Тестовая, 1", price_rub=5_000_000, area_m2=50.0, rooms=2, house_source="cian_newbuilding" if nb else "cian", house_ext_id="99999" if nb else None, ) class _FakeRuns: """Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel.""" def __init__(self, *, cancel_at_anchor: int | None = None) -> None: self.cancel_at_anchor = cancel_at_anchor self._is_cancelled_calls = 0 self.heartbeats: list[dict[str, int]] = [] self.done: dict[str, int] | None = None self.failed: tuple[str, dict[str, int]] | None = None self.banned: tuple[str, dict[str, int]] | None = None def is_cancelled(self, _db: Any, _run_id: int) -> bool: self._is_cancelled_calls += 1 if self.cancel_at_anchor is None: return False return self._is_cancelled_calls >= self.cancel_at_anchor def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: self.heartbeats.append(dict(counters)) def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: self.done = dict(counters) def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: self.failed = (error, dict(counters)) def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: self.banned = (error, dict(counters)) @pytest.fixture(autouse=True) def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None: """asyncio.sleep → instant (no test delays).""" async def _instant(_secs: float) -> None: return None monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant) @pytest.fixture(autouse=True) def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None: """Нейтрализуем CianScraper.__init__/__aenter__/__aexit__ — нет DB вызовов.""" def _init(self: CianScraper) -> None: self.request_delay_sec = 0.0 self._cffi = None async def _aenter(self: CianScraper) -> CianScraper: return self async def _aexit(self: CianScraper, *_args: Any) -> None: return None monkeypatch.setattr(CianScraper, "__init__", _init) monkeypatch.setattr(CianScraper, "__aenter__", _aenter) monkeypatch.setattr(CianScraper, "__aexit__", _aexit) def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None: monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled) monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned) # ── CianCitySweepCounters ─────────────────────────────────────────────────── def test_cian_sweep_counters_defaults() -> None: from app.services.scrape_pipeline import CianCitySweepCounters c = CianCitySweepCounters() assert c.anchors_total == 0 assert c.lots_fetched == 0 assert c.detail_attempted == 0 assert c.houses_attempted == 0 assert c.errors_count == 0 def test_cian_sweep_counters_to_dict_all_keys() -> None: from dataclasses import fields from app.services.scrape_pipeline import CianCitySweepCounters c = CianCitySweepCounters() d = c.to_dict() expected = {f.name for f in fields(c)} assert set(d.keys()) == expected # ── Full sweep — фазы и агрегация ────────────────────────────────────────── async def test_sweep_iterates_all_anchors_serp_and_save( monkeypatch: pytest.MonkeyPatch, ) -> None: """SERP вызывается для каждого anchor, save_listings получает накопленные lots.""" serp_calls: list[str] = [] save_calls: list[int] = [] async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8): serp_calls.append(f"{lat:.4f}") return [_fake_lot(f"{lat}-{i}") for i in range(3)] def fake_save(_db: Any, lots: list, *, run_id: int | None = None): save_calls.append(len(lots)) return len(lots), 0 monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi) monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save) # Stub detail + houses fetchers (возвращают None → graceful skip) monkeypatch.setattr( "app.services.scrapers.cian_detail.fetch_detail", AsyncMock(return_value=None), ) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_cian_city_sweep( db=MagicMock(), run_id=1, anchors=TEST_ANCHORS, pages_per_anchor=2, detail_top_n=0, # skip detail phase enrich_houses=False, request_delay_sec=0.0, ) assert len(serp_calls) == len(TEST_ANCHORS) assert len(save_calls) == len(TEST_ANCHORS) assert counters.anchors_done == len(TEST_ANCHORS) assert counters.lots_fetched == len(TEST_ANCHORS) * 3 assert counters.lots_inserted == len(TEST_ANCHORS) * 3 assert counters.errors_count == 0 assert fake.done is not None assert fake.done["lots_fetched"] == len(TEST_ANCHORS) * 3 assert fake.failed is None assert fake.banned is None async def test_detail_phase_runs_and_fills_counters( monkeypatch: pytest.MonkeyPatch, ) -> None: """detail-фаза вызывается и счётчики заполняются.""" from app.services.scrapers.cian_detail import DetailEnrichment async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8): return [_fake_lot(f"{lat}-{i}") for i in range(2)] monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) # DB mock returning 2 rows for detail query mock_db = MagicMock() mock_db.execute.return_value.mappings.return_value.all.return_value = [ {"id": 101, "source_url": "https://ekb.cian.ru/sale/flat/101/"}, {"id": 102, "source_url": "https://ekb.cian.ru/sale/flat/102/"}, ] fake_enrichment = DetailEnrichment(cian_id=101) mock_fetch_detail = AsyncMock(return_value=fake_enrichment) mock_save_detail = MagicMock() monkeypatch.setattr( "app.services.scrapers.cian_detail.fetch_detail", mock_fetch_detail, ) monkeypatch.setattr( "app.services.scrapers.cian_detail.save_detail_enrichment", mock_save_detail, ) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_cian_city_sweep( db=mock_db, run_id=2, anchors=TEST_ANCHORS[:1], # один anchor detail_top_n=2, enrich_houses=False, request_delay_sec=0.0, ) assert counters.detail_attempted == 2 assert counters.detail_enriched == 2 assert counters.detail_failed == 0 assert mock_fetch_detail.call_count == 2 assert mock_save_detail.call_count == 2 assert fake.done is not None async def test_enrich_houses_false_skips_houses_phase( monkeypatch: pytest.MonkeyPatch, ) -> None: """enrich_houses=False → houses-фаза не выполняется.""" nb_fetch_calls: list[str] = [] async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8): return [_fake_lot("1001", nb=True)] async def fake_fetch_nb(url: str, **_: Any) -> None: nb_fetch_calls.append(url) return None monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) monkeypatch.setattr( "app.services.scrapers.cian_newbuilding.fetch_newbuilding", fake_fetch_nb, ) monkeypatch.setattr( "app.services.scrapers.cian_detail.fetch_detail", AsyncMock(return_value=None), ) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_cian_city_sweep( db=MagicMock(), run_id=3, anchors=TEST_ANCHORS[:1], detail_top_n=0, enrich_houses=False, request_delay_sec=0.0, ) assert len(nb_fetch_calls) == 0 assert counters.houses_attempted == 0 assert counters.houses_enriched == 0 assert fake.done is not None async def test_cancel_midway_stops_remaining( monkeypatch: pytest.MonkeyPatch, ) -> None: """Cooperative cancel перед anchor #2 → anchor #1 выполняется, #2+ нет.""" serp_calls: list[str] = [] async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8): serp_calls.append(f"{lat:.4f}") return [_fake_lot(f"{lat}-1")] monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) monkeypatch.setattr( "app.services.scrapers.cian_detail.fetch_detail", AsyncMock(return_value=None), ) fake = _FakeRuns(cancel_at_anchor=2) # cancel при is_cancelled_calls >= 2 → перед anchor #2 _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_cian_city_sweep( db=MagicMock(), run_id=4, anchors=TEST_ANCHORS, detail_top_n=0, enrich_houses=False, request_delay_sec=0.0, ) assert len(serp_calls) == 1, "только anchor #1 должен выполниться" assert counters.anchors_done == 1 assert fake.done is None assert fake.banned is None async def test_consecutive_serp_failures_abort_with_banned( monkeypatch: pytest.MonkeyPatch, ) -> None: """N подряд SERP-ошибок → mark_banned + abort sweep.""" from app.services.scrape_pipeline import CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES attempt_count = 0 async def fake_fetch_fail(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8): nonlocal attempt_count attempt_count += 1 raise RuntimeError("cian blocked") monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_fail) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) fake = _FakeRuns() _install_runs(monkeypatch, fake) # Передаём больше anchor'ов чем CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES anchors = [*TEST_ANCHORS, (56.77, 60.55, "A4"), (56.865, 60.62, "A5")] counters = await scrape_pipeline.run_cian_city_sweep( db=MagicMock(), run_id=5, anchors=anchors, detail_top_n=0, enrich_houses=False, request_delay_sec=0.0, ) assert attempt_count == CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES assert counters.errors_count == CIAN_SWEEP_MAX_CONSECUTIVE_FAILURES assert fake.banned is not None, "mark_banned должен быть вызван" assert "consecutive" in fake.banned[0] assert fake.done is None async def test_per_item_detail_error_does_not_abort_sweep( monkeypatch: pytest.MonkeyPatch, ) -> None: """Ошибка при detail-обогащении одного листинга не валит sweep.""" call_count = 0 async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8): return [_fake_lot(f"lot-{idx}") for idx in range(2)] monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) mock_db = MagicMock() # Возвращаем 2 listing-а для detail mock_db.execute.return_value.mappings.return_value.all.return_value = [ {"id": 201, "source_url": "https://ekb.cian.ru/sale/flat/201/"}, {"id": 202, "source_url": "https://ekb.cian.ru/sale/flat/202/"}, ] async def fetch_detail_sometimes_fails(url: str, **_: Any): nonlocal call_count call_count += 1 if call_count == 1: raise RuntimeError("transient network error") from app.services.scrapers.cian_detail import DetailEnrichment return DetailEnrichment(cian_id=202) monkeypatch.setattr( "app.services.scrapers.cian_detail.fetch_detail", fetch_detail_sometimes_fails, ) monkeypatch.setattr( "app.services.scrapers.cian_detail.save_detail_enrichment", MagicMock(), ) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_cian_city_sweep( db=mock_db, run_id=6, anchors=TEST_ANCHORS[:1], detail_top_n=2, enrich_houses=False, request_delay_sec=0.0, ) assert counters.detail_attempted == 2 assert counters.detail_failed == 1 assert counters.detail_enriched == 1 assert counters.errors_count >= 1 assert fake.done is not None, "sweep должен завершиться mark_done несмотря на ошибку" assert fake.banned is None async def test_mark_done_always_called( monkeypatch: pytest.MonkeyPatch, ) -> None: """mark_done вызывается даже при пустом SERP.""" async def fake_fetch_empty(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8): return [] monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_empty) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) monkeypatch.setattr( "app.services.scrapers.cian_detail.fetch_detail", AsyncMock(return_value=None), ) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_cian_city_sweep( db=MagicMock(), run_id=7, anchors=TEST_ANCHORS, detail_top_n=0, enrich_houses=False, request_delay_sec=0.0, ) assert fake.done is not None assert counters.lots_fetched == 0 assert counters.anchors_done == len(TEST_ANCHORS) async def test_houses_phase_uses_house_sources_join( monkeypatch: pytest.MonkeyPatch, ) -> None: """Houses-фаза формирует JOIN на house_sources, не h.ext_source напрямую.""" async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8): # Один lot с house_source="cian_newbuilding" → попадёт в houses-фазу return [_fake_lot("nb-1", nb=True)] monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) monkeypatch.setattr( "app.services.scrapers.cian_detail.fetch_detail", AsyncMock(return_value=None), ) monkeypatch.setattr( "app.services.scrapers.cian_newbuilding.fetch_newbuilding", AsyncMock(return_value=None), ) monkeypatch.setattr( "app.services.scrapers.cian_newbuilding.save_newbuilding_enrichment", MagicMock(), ) mock_db = MagicMock() # DB execute → возвращает одну house-строку mock_db.execute.return_value.mappings.return_value.all.return_value = [ {"id": 501, "cian_zhk_url": "https://ekb.cian.ru/kupit-kvartiru/zhk-test/"}, ] fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_cian_city_sweep( db=mock_db, run_id=10, anchors=TEST_ANCHORS[:1], detail_top_n=0, enrich_houses=True, request_delay_sec=0.0, ) # Проверяем SQL — должен содержать JOIN house_sources, а не h.ext_source assert mock_db.execute.called executed_sql_args = [str(c.args[0]) for c in mock_db.execute.call_args_list] houses_sql_calls = [s for s in executed_sql_args if "house" in s.lower()] assert houses_sql_calls, "должен быть хотя бы один SQL-вызов связанный с houses" houses_query = houses_sql_calls[-1] assert ( "house_sources" in houses_query ), f"запрос должен JOIN house_sources, а не обращаться к h.ext_source: {houses_query!r}" assert "ext_source" not in houses_query.split("house_sources")[0] or True # tolerate join expr # Главное: нет обращения к h.ext_source (старый баг) assert ( "h.ext_source" not in houses_query ), f"колонка h.ext_source не существует в houses: {houses_query!r}" assert ( "h.ext_id" not in houses_query ), f"колонка h.ext_id не существует в houses: {houses_query!r}" assert fake.done is not None, "mark_done должен быть вызван" assert fake.failed is None, "mark_failed НЕ должен быть вызван" assert counters.houses_attempted == 1 async def test_houses_db_query_error_graceful_mark_done( monkeypatch: pytest.MonkeyPatch, ) -> None: """Ошибка SQL в houses-фазе (напр. UndefinedColumn) → graceful: mark_done, НЕ mark_failed, houses_failed увеличивается.""" async def fake_fetch_multi(self: CianScraper, lat: float, lon: float, r: int, pages: int = 8): # Два nb-лота → houses-фаза попытается запросить 2 id return [_fake_lot("nb-10", nb=True), _fake_lot("nb-11", nb=True)] monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) monkeypatch.setattr( "app.services.scrapers.cian_detail.fetch_detail", AsyncMock(return_value=None), ) mock_db = MagicMock() def _db_execute_raises(sql, params=None): sql_text = str(sql) if "house" in sql_text.lower() and "cian_newbuilding" in sql_text: raise RuntimeError("column h.ext_source does not exist") result = MagicMock() result.mappings.return_value.all.return_value = [] return result mock_db.execute.side_effect = _db_execute_raises fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_cian_city_sweep( db=mock_db, run_id=11, anchors=TEST_ANCHORS[:1], detail_top_n=0, enrich_houses=True, request_delay_sec=0.0, ) assert fake.done is not None, "mark_done должен быть вызван даже при ошибке houses-фазы" assert fake.failed is None, "mark_failed НЕ должен быть вызван из-за ошибки houses-фазы" assert counters.houses_failed > 0, "houses_failed должен вырасти при ошибке" assert counters.houses_attempted == 0, "houses_attempted не растёт — до него не дошли" # ── Admin endpoint tests (offline) ────────────────────────────────────────── @pytest.fixture def app_with_admin(): from unittest.mock import MagicMock from fastapi import FastAPI from fastapi.testclient import TestClient from app.api.v1 import admin as admin_module from app.core.db import get_db app = FastAPI() app.include_router(admin_module.router, prefix="/api/v1/admin") def fake_db(): yield MagicMock() app.dependency_overrides[get_db] = fake_db return TestClient(app) def test_cian_start_endpoint_validates_pages_per_anchor(app_with_admin) -> None: """pages_per_anchor=99 превышает max=10 → 422.""" r = app_with_admin.post( "/api/v1/admin/scrape/cian-city-sweep", json={"pages_per_anchor": 99}, ) assert r.status_code == 422 def test_cian_start_endpoint_validates_delay_too_low(app_with_admin) -> None: """request_delay_sec=1.0 меньше min=3.0 → 422.""" r = app_with_admin.post( "/api/v1/admin/scrape/cian-city-sweep", json={"request_delay_sec": 1.0}, ) assert r.status_code == 422 def test_cian_start_endpoint_ok(app_with_admin) -> None: """Valid request → 200, run_id в ответе.""" from unittest.mock import AsyncMock, patch with ( patch("app.services.scrape_runs.create_run", return_value=42), patch("app.services.scrape_pipeline.run_cian_city_sweep", new_callable=AsyncMock), ): r = app_with_admin.post( "/api/v1/admin/scrape/cian-city-sweep", json={"pages_per_anchor": 2, "detail_top_n": 5, "enrich_houses": False}, ) assert r.status_code == 200 body = r.json() assert body["run_id"] == 42 assert body["status"] == "running" assert body["pages_per_anchor"] == 2 assert body["detail_top_n"] == 5 def test_cian_cancel_endpoint(app_with_admin) -> None: """Cancel endpoint возвращает run_id + cancelled=True.""" from unittest.mock import patch with patch("app.services.scrape_runs.mark_cancelled", return_value=True): r = app_with_admin.post("/api/v1/admin/scrape/cian-city-sweep/77/cancel") assert r.status_code == 200 body = r.json() assert body["run_id"] == 77 assert body["cancelled"] is True assert body["ok"] is True def test_cian_cancel_endpoint_not_running(app_with_admin) -> None: """Cancel на не-running sweep → cancelled=False.""" from unittest.mock import patch with patch("app.services.scrape_runs.mark_cancelled", return_value=False): r = app_with_admin.post("/api/v1/admin/scrape/cian-city-sweep/88/cancel") assert r.status_code == 200 assert r.json()["cancelled"] is False def test_cian_list_runs_endpoint(app_with_admin) -> None: """List runs endpoint возвращает список.""" from datetime import UTC, datetime from unittest.mock import patch fake_rows = [ { "run_id": 10, "source": "cian_city_sweep", "status": "done", "params": {"pages_per_anchor": 3}, "counters": {"lots_fetched": 150}, "error": None, "started_at": datetime(2025, 5, 1, tzinfo=UTC), "finished_at": datetime(2025, 5, 1, 1, tzinfo=UTC), "heartbeat_at": datetime(2025, 5, 1, 1, tzinfo=UTC), } ] with patch("app.services.scrape_runs.list_recent", return_value=fake_rows): r = app_with_admin.get("/api/v1/admin/scrape/cian-city-sweep/runs") assert r.status_code == 200 body = r.json() assert len(body) == 1 assert body[0]["run_id"] == 10 assert body[0]["source"] == "cian_city_sweep" assert "2025-05-01" in body[0]["started_at"]