"""Offline unit tests for run_domclick_city_sweep + DomClick parser. Pure & fast: NO live network, NO DB. Mirrors test_cian_city_sweep orchestration semantics, but DomClick is CITYWIDE (no anchor-loop) и SERP-only (no detail/houses). Coverage: - fetch_city вызывается, save_listings получает накопленные lots с source='domklik' - DomClickCitySweepCounters корректно заполняется (lots_fetched/inserted/updated) - Cooperative cancel прерывает sweep перед SERP - fetch_city exception не валит sweep (graceful, errors_count++, mark_done) - _parse_html: минимальная DomClick карточка → ScrapedLot с rooms/area/floor/price """ from __future__ import annotations import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from typing import Any from unittest.mock import MagicMock import pytest from app.services import scrape_pipeline from app.services.scrapers.base import ScrapedLot from app.services.scrapers.domclick import DomClickScraper def _fake_lot(offer_id: str) -> ScrapedLot: return ScrapedLot( source="domklik", source_url=f"https://domclick.ru/card/sale__flat__{offer_id}", source_id=offer_id, address="Екатеринбург, улица Тестовая, 1", price_rub=5_000_000, area_m2=50.0, rooms=2, lat=None, lon=None, ) class _FakeRuns: """Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel.""" def __init__(self, *, cancel: bool = False) -> None: self.cancel = cancel self.heartbeats: list[dict[str, int]] = [] self.done: dict[str, int] | None = None self.failed: tuple[str, dict[str, int]] | None = None def is_cancelled(self, _db: Any, _run_id: int) -> bool: return self.cancel def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: self.heartbeats.append(dict(counters)) def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: self.done = dict(counters) def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: self.failed = (error, dict(counters)) @pytest.fixture(autouse=True) def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None: async def _instant(_secs: float) -> None: return None monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant) @pytest.fixture(autouse=True) def _stub_domclick_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None: """Нейтрализуем DomClickScraper.__init__/__aenter__/__aexit__ — нет DB/browser.""" def _init(self: DomClickScraper) -> None: self.request_delay_sec = 0.0 self.parse_failures = 0 self.geo_filtered = 0 self.blocked = False self.fetch_errors = 0 async def _aenter(self: DomClickScraper) -> DomClickScraper: return self async def _aexit(self: DomClickScraper, *_args: Any) -> None: return None monkeypatch.setattr(DomClickScraper, "__init__", _init) monkeypatch.setattr(DomClickScraper, "__aenter__", _aenter) monkeypatch.setattr(DomClickScraper, "__aexit__", _aexit) def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None: monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled) monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed) # ── DomClickCitySweepCounters ─────────────────────────────────────────────── def test_domclick_sweep_counters_defaults() -> None: from app.services.scrape_pipeline import DomClickCitySweepCounters c = DomClickCitySweepCounters() assert c.lots_fetched == 0 assert c.lots_inserted == 0 assert c.lots_updated == 0 assert c.pages_fetched == 0 assert c.errors_count == 0 assert c.blocked == 0 assert c.geo_filtered == 0 def test_domclick_sweep_counters_to_dict_all_keys() -> None: from dataclasses import fields from app.services.scrape_pipeline import DomClickCitySweepCounters c = DomClickCitySweepCounters() d = c.to_dict() expected = {f.name for f in fields(c)} assert set(d.keys()) == expected # ── Full sweep — фазы и агрегация ────────────────────────────────────────── async def test_sweep_fetches_and_saves_with_source(monkeypatch: pytest.MonkeyPatch) -> None: """fetch_city → save_listings получает 2 lots с source='domklik', counters заполнены.""" fetch_calls: list[tuple[int, int]] = [] save_calls: list[list[ScrapedLot]] = [] async def fake_fetch_city( self: DomClickScraper, city_id: int, rooms: list[int] | None = None, pages: int = 20, ) -> list[ScrapedLot]: fetch_calls.append((city_id, pages)) return [_fake_lot("100"), _fake_lot("101")] def fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]: save_calls.append(list(lots)) return len(lots), 0 monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_city) monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_domclick_city_sweep( db=MagicMock(), run_id=1, city_id=4, rooms=[1, 2], pages=3, request_delay_sec=0.0, ) assert len(fetch_calls) == 1 assert fetch_calls[0] == (4, 3) assert len(save_calls) == 1 assert len(save_calls[0]) == 2 assert all(lot.source == "domklik" for lot in save_calls[0]) assert counters.lots_fetched == 2 assert counters.lots_inserted == 2 assert counters.lots_updated == 0 assert counters.errors_count == 0 assert fake.done is not None assert fake.done["lots_fetched"] == 2 assert fake.failed is None async def test_sweep_empty_serp_still_mark_done(monkeypatch: pytest.MonkeyPatch) -> None: """Пустой SERP → save не вызывается, но mark_done всё равно вызывается.""" save_called = False async def fake_fetch_empty( self: DomClickScraper, city_id: int, rooms: list[int] | None = None, pages: int = 20, ) -> list[ScrapedLot]: return [] def fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]: nonlocal save_called save_called = True return 0, 0 monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_empty) monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_domclick_city_sweep( db=MagicMock(), run_id=2, request_delay_sec=0.0 ) assert save_called is False assert counters.lots_fetched == 0 assert fake.done is not None async def test_cancel_before_serp_skips_fetch(monkeypatch: pytest.MonkeyPatch) -> None: """Cooperative cancel перед SERP → fetch_city НЕ вызывается, mark_done НЕ вызывается.""" fetch_called = False async def fake_fetch( self: DomClickScraper, city_id: int, rooms: list[int] | None = None, pages: int = 20, ) -> list[ScrapedLot]: nonlocal fetch_called fetch_called = True return [_fake_lot("1")] monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch) monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0)) fake = _FakeRuns(cancel=True) _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_domclick_city_sweep( db=MagicMock(), run_id=3, request_delay_sec=0.0 ) assert fetch_called is False assert counters.lots_fetched == 0 assert fake.done is None async def test_fetch_city_exception_no_reraise(monkeypatch: pytest.MonkeyPatch) -> None: """Исключение в fetch_city → errors_count++, sweep НЕ ре-райзит (graceful). Honest-status (#1968): полный крах SERP-фазы (errors_count=1, 0 lots) → mark_failed, НЕ mark_done. Run без единого листинга по внешней причине — это реальный failure, а не успешный пустой прогон. Re-raise при этом НЕ происходит (task завершается чисто). """ async def fake_fetch_fail( self: DomClickScraper, city_id: int, rooms: list[int] | None = None, pages: int = 20, ) -> list[ScrapedLot]: raise RuntimeError("browser crashed") monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_fail) monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0)) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_domclick_city_sweep( db=MagicMock(), run_id=4, request_delay_sec=0.0 ) assert counters.errors_count == 1 assert counters.lots_fetched == 0 # 0 lots + error → honest mark_failed (не маскируем как done) assert fake.failed is not None assert fake.done is None async def test_fetch_errors_and_no_lots_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None: """Scraper fetch_errors>0 + 0 лотов (без block) → mark_failed (FIX 3 / closes #1968). Нераспознанный block-вариант приходит как fetch_error (не blocked=True), но honest-status всё равно обязан пометить run failed при пустом результате. """ async def fake_fetch_errs( self: DomClickScraper, city_id: int, rooms: list[int] | None = None, pages: int = 100, ) -> list[ScrapedLot]: # Имитируем не-block ошибки извлечения JSON: blocked остаётся False self.fetch_errors = 3 return [] monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_errs) monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0)) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_domclick_city_sweep( db=MagicMock(), run_id=6, request_delay_sec=0.0 ) assert counters.lots_fetched == 0 assert counters.blocked == 0 assert counters.errors_count >= 3 # fetch_errors свёрнуты в errors_count assert fake.failed is not None assert fake.done is None async def test_blocked_and_no_lots_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None: """QRATOR-блок + 0 лотов → mark_failed (честный статус, closes #1968).""" async def fake_fetch_blocked( self: DomClickScraper, city_id: int, rooms: list[int] | None = None, pages: int = 100, ) -> list[ScrapedLot]: # Имитируем QRATOR-блок: установим флаг на scraper и вернём пустой список self.blocked = True return [] monkeypatch.setattr(DomClickScraper, "fetch_city", fake_fetch_blocked) monkeypatch.setattr(scrape_pipeline, "save_listings", lambda *a, **k: (0, 0)) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_domclick_city_sweep( db=MagicMock(), run_id=5, request_delay_sec=0.0 ) assert counters.lots_fetched == 0 assert counters.blocked == 1 # Честный статус: mark_failed должен быть вызван, mark_done — нет assert fake.failed is not None assert "QRATOR" in fake.failed[0] assert fake.done is None # ── _map_item via sweep — quick integration smoke ──────────────────────────── def test_map_item_basic_mapping() -> None: """DomClickScraper._map_item маппит BFF offer-item → ScrapedLot корректно.""" scraper = DomClickScraper() # __init__ stubbed item = { "id": 111222333, "path": "https://domclick.ru/card/sale__flat__111222333", "location": {"lat": 56.838, "lon": 60.612}, "address": {"displayName": "Екатеринбург, улица Ленина, 10"}, "objectInfo": {"area": 52.3, "rooms": 2, "floor": 5}, "house": {"floors": 16, "buildYear": 2010}, "price": 6_500_000, "squarePrice": 124282, "flatComplex": None, "isRosreestrApproved": False, "publishedDate": "2026-06-01T10:00:00+05:00", "updatedDate": None, "lastPriceHistoryState": None, "offerRegionName": "Екатеринбург", } lot = scraper._map_item(item) assert lot is not None assert lot.source == "domklik" assert lot.source_id == "111222333" assert lot.rooms == 2 assert lot.area_m2 == pytest.approx(52.3) assert lot.floor == 5 assert lot.total_floors == 16 assert lot.price_rub == 6_500_000 assert lot.listing_segment == "vtorichka" assert lot.lat == pytest.approx(56.838) assert lot.lon == pytest.approx(60.612)