"""Offline unit tests for run_n1_city_sweep (#575). Pure & fast: NO live network, NO DB. Mirrors test_yandex_city_sweep — monkeypatch N1Scraper lifecycle + fetch_around, scrape_pipeline.save_listings, scrape_runs.*. Asserts orchestration semantics only (live N1 behavior verified by post-deploy QA). NB: run_n1_city_sweep пагинирует page=1..pages_per_anchor (1-based), в отличие от yandex (0-based) — тесты учитывают это. """ import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from typing import Any import pytest from app.services import scrape_pipeline from app.services.scrapers.base import ScrapedLot from app.services.scrapers.n1 import N1Scraper TEST_ANCHORS: list[tuple[float, float, str]] = [ (56.8400, 60.6050, "A1"), (56.7950, 60.5300, "A2"), (56.8970, 60.6100, "A3"), ] def _fake_lot(offer_id: str) -> ScrapedLot: return ScrapedLot( source="n1", source_url=f"https://ekb.n1.ru/view/{offer_id}/", source_id=offer_id, address="Екатеринбург, улица Тестовая, 1", price_rub=5_000_000, area_m2=50.0, rooms=2, ) class _FakeRuns: def __init__(self, *, cancel_at_anchor: int | None = None) -> None: self.cancel_at_anchor = cancel_at_anchor self.is_cancelled_calls = 0 self.heartbeats: list[dict[str, int]] = [] self.done: dict[str, int] | None = None self.failed: tuple[str, dict[str, int]] | None = None self.banned: tuple[str, dict[str, int]] | None = None def is_cancelled(self, _db: Any, _run_id: int) -> bool: self.is_cancelled_calls += 1 if self.cancel_at_anchor is None: return False return self.is_cancelled_calls >= self.cancel_at_anchor def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: self.heartbeats.append(dict(counters)) def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: self.done = dict(counters) def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: self.failed = (error, dict(counters)) def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: self.banned = (error, dict(counters)) @pytest.fixture(autouse=True) def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None: async def _instant(_secs: float) -> None: return None monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant) @pytest.fixture(autouse=True) def _stub_scraper_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None: """Neutralize __init__/__aenter__/__aexit__ — no get_scraper_delay DB, no curl_cffi.""" def _init(self: N1Scraper) -> None: self.request_delay_sec = 0.0 self._cffi = None async def _aenter(self: N1Scraper) -> N1Scraper: return self async def _aexit(self: N1Scraper, *_args: Any) -> None: return None monkeypatch.setattr(N1Scraper, "__init__", _init) monkeypatch.setattr(N1Scraper, "__aenter__", _aenter) monkeypatch.setattr(N1Scraper, "__aexit__", _aexit) def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None: monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled) monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed) monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned) async def test_sweep_iterates_all_anchors_and_aggregates(monkeypatch: pytest.MonkeyPatch) -> None: fetch_calls: list[tuple[float, float, int]] = [] save_calls: list[int] = [] async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1): fetch_calls.append((lat, lon, page)) return [_fake_lot(f"{lat}-{page}-{i}") for i in range(2)] def fake_save(_db, lots, *, run_id=None): save_calls.append(len(lots)) return len(lots), 0 monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch) monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_n1_city_sweep( db=object(), run_id=1, anchors=TEST_ANCHORS, pages_per_anchor=2, request_delay_sec=0.0 ) assert len(fetch_calls) == len(TEST_ANCHORS) * 2 assert [p for _, _, p in fetch_calls[:2]] == [1, 2] # 1-based pagination assert len(save_calls) == len(TEST_ANCHORS) assert counters.anchors_done == len(TEST_ANCHORS) assert counters.pages_fetched == len(TEST_ANCHORS) * 2 assert counters.lots_fetched == 12 assert counters.lots_inserted == 12 assert counters.errors_count == 0 assert fake.done is not None and fake.done["lots_fetched"] == 12 assert fake.failed is None and fake.banned is None async def test_save_listings_receives_run_id(monkeypatch: pytest.MonkeyPatch) -> None: seen: list[int | None] = [] async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1): return [_fake_lot(f"{lat}-{page}")] def fake_save(_db, lots, *, run_id=None): seen.append(run_id) return len(lots), 0 monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch) monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save) _install_runs(monkeypatch, _FakeRuns()) await scrape_pipeline.run_n1_city_sweep( db=object(), run_id=777, anchors=TEST_ANCHORS, pages_per_anchor=1 ) assert seen == [777, 777, 777] async def test_empty_page_stops_pagination(monkeypatch: pytest.MonkeyPatch) -> None: pages_seen: list[int] = [] async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1): pages_seen.append(page) if page == 1: return [_fake_lot(f"{lat}-1")] return [] # page 2 empty → stop monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) fake = _FakeRuns() _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_n1_city_sweep( db=object(), run_id=2, anchors=TEST_ANCHORS, pages_per_anchor=5 ) assert pages_seen == [1, 2, 1, 2, 1, 2] assert counters.pages_fetched == 6 assert counters.lots_fetched == 3 assert counters.anchors_done == 3 async def test_cancel_midway_stops_remaining(monkeypatch: pytest.MonkeyPatch) -> None: fetch_calls: list[str] = [] async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1): fetch_calls.append(f"{lat}") return [_fake_lot(f"{lat}-{page}")] monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) fake = _FakeRuns(cancel_at_anchor=2) _install_runs(monkeypatch, fake) counters = await scrape_pipeline.run_n1_city_sweep( db=object(), run_id=3, anchors=TEST_ANCHORS, pages_per_anchor=1 ) assert len(fetch_calls) == 1 assert counters.anchors_done == 1 assert fake.done is None and fake.banned is None async def test_consecutive_failures_abort_sweep(monkeypatch: pytest.MonkeyPatch) -> None: attempts: list[str] = [] async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1): attempts.append(f"{lat}") raise RuntimeError("n1 blocked") monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch) monkeypatch.setattr( scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0) ) fake = _FakeRuns() _install_runs(monkeypatch, fake) anchors = [*TEST_ANCHORS, (56.77, 60.55, "A4"), (56.865, 60.62, "A5")] counters = await scrape_pipeline.run_n1_city_sweep( db=object(), run_id=5, anchors=anchors, pages_per_anchor=1 ) n = scrape_pipeline.N1_SWEEP_MAX_CONSECUTIVE_FAILURES assert len(attempts) == n assert counters.errors_count == n assert counters.anchors_done == n assert fake.banned is not None and "consecutive" in fake.banned[0] assert fake.done is None