gendesign/tradein-mvp/backend/tests/test_yandex_city_sweep.py

"""Offline unit tests for run_yandex_city_sweep (#561, shipped DORMANT).

Pure & fast: NO live network, NO DB. We monkeypatch:
  - YandexRealtyScraper.__init__ / __aenter__ / __aexit__ — avoid get_scraper_delay
    DB read + real httpx.AsyncClient creation.
  - YandexRealtyScraper.fetch_around — return a fixed list of fake ScrapedLot.
  - scrape_pipeline.save_listings — counter/no-op (no real listings table).
  - scrape_runs.* (is_cancelled / update_heartbeat / mark_*) — in-memory fakes.

These tests assert orchestration semantics only; live Yandex behavior is intentionally
unverified (sweep is dormant until egress-IP rotation #623).
"""
import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from typing import Any

import pytest

from app.services import scrape_pipeline
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.yandex_realty import YandexRealtyScraper

# 3 anchors keep tests fast and let us probe "anchor k cancels/fails, others proceed".
TEST_ANCHORS: list[tuple[float, float, str]] = [
    (56.8400, 60.6050, "A1"),
    (56.7950, 60.5300, "A2"),
    (56.8970, 60.6100, "A3"),
]


def _fake_lot(offer_id: str) -> ScrapedLot:
    """Minimal valid ScrapedLot (source/source_url required, price_rub > 0)."""
    return ScrapedLot(
        source="yandex",
        source_url=f"https://realty.yandex.ru/offer/{offer_id}/",
        source_id=offer_id,
        address="Екатеринбург, улица Тестовая, 1",
        price_rub=5_000_000,
        area_m2=50.0,
        rooms=2,
    )


class _FakeRuns:
    """In-memory stand-in for app.services.scrape_runs (no DB)."""

    def __init__(self, *, cancel_at_anchor: int | None = None) -> None:
        # cancel_at_anchor: 1-based anchor index at which is_cancelled flips True.
        self.cancel_at_anchor = cancel_at_anchor
        self.is_cancelled_calls = 0
        self.heartbeats: list[dict[str, int]] = []
        self.done: dict[str, int] | None = None
        self.failed: tuple[str, dict[str, int]] | None = None
        self.banned: tuple[str, dict[str, int]] | None = None

    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
        self.is_cancelled_calls += 1
        if self.cancel_at_anchor is None:
            return False
        return self.is_cancelled_calls >= self.cancel_at_anchor

    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.heartbeats.append(dict(counters))

    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.done = dict(counters)

    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.failed = (error, dict(counters))

    def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.banned = (error, dict(counters))


@pytest.fixture(autouse=True)
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
    """Kill inter-anchor asyncio.sleep so tests are instant."""
    async def _instant(_secs: float) -> None:
        return None

    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)


@pytest.fixture(autouse=True)
def _stub_scraper_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
    """Neutralize __init__/__aenter__/__aexit__ — no get_scraper_delay DB, no httpx."""

    def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
        self.city = city
        self.request_delay_sec = 0.0
        self._client = None

    async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
        return self

    async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
        return None

    monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
    monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
    monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)


def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)


# ── Happy path: all anchors iterated, counters aggregated ───────────────────


async def test_sweep_iterates_all_anchors_and_aggregates(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    fetch_calls: list[tuple[float, float, int]] = []
    save_calls: list[int] = []

    async def fake_fetch(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        page: int = 0,
    ) -> list[ScrapedLot]:
        fetch_calls.append((lat, lon, page))
        # 2 lots per page; let pagination run the full pages_per_anchor.
        return [_fake_lot(f"{lat}-{page}-{i}") for i in range(2)]

    def fake_save(
        _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
    ) -> tuple[int, int]:
        save_calls.append(len(lots))
        # Pretend everything is a fresh insert.
        return len(lots), 0

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=object(),
        run_id=1,
        anchors=TEST_ANCHORS,
        pages_per_anchor=2,
        request_delay_sec=0.0,
    )

    # fetch_around called pages_per_anchor times per anchor (no empty-page short-circuit).
    assert len(fetch_calls) == len(TEST_ANCHORS) * 2
    # save_listings called once per anchor (anchor-level batch).
    assert len(save_calls) == len(TEST_ANCHORS)
    # Counters: 3 anchors × 2 pages × 2 lots = 12.
    assert counters.anchors_total == len(TEST_ANCHORS)
    assert counters.anchors_done == len(TEST_ANCHORS)
    assert counters.pages_fetched == len(TEST_ANCHORS) * 2
    assert counters.lots_fetched == 12
    assert counters.lots_inserted == 12
    assert counters.lots_updated == 0
    assert counters.errors_count == 0
    # mark_done called with final counters; no failure/ban.
    assert fake.done is not None
    assert fake.done["lots_fetched"] == 12
    assert fake.failed is None
    assert fake.banned is None


async def test_save_listings_receives_run_id(monkeypatch: pytest.MonkeyPatch) -> None:
    """run_id must be threaded into save_listings for snapshot provenance."""
    seen_run_ids: list[int | None] = []

    async def fake_fetch(self: YandexRealtyScraper, lat, lon, radius_m=1000, page=0):
        return [_fake_lot(f"{lat}-{page}")]

    def fake_save(_db, lots, *, run_id=None):
        seen_run_ids.append(run_id)
        return len(lots), 0

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
    _install_runs(monkeypatch, _FakeRuns())

    await scrape_pipeline.run_yandex_city_sweep(
        db=object(), run_id=777, anchors=TEST_ANCHORS, pages_per_anchor=1,
    )
    assert seen_run_ids == [777, 777, 777]


# ── Empty page short-circuits pagination ────────────────────────────────────


async def test_empty_page_stops_pagination(monkeypatch: pytest.MonkeyPatch) -> None:
    """An empty SERP page ends pagination for that anchor (no further pages)."""
    pages_seen: list[int] = []

    async def fake_fetch(self: YandexRealtyScraper, lat, lon, radius_m=1000, page=0):
        pages_seen.append(page)
        if page == 0:
            return [_fake_lot(f"{lat}-0")]
        return []  # page 1 empty → stop

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(
        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
    )
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=object(), run_id=2, anchors=TEST_ANCHORS, pages_per_anchor=5,
    )
    # Per anchor: page 0 (1 lot) then page 1 (empty, break) → 2 pages each.
    assert pages_seen == [0, 1, 0, 1, 0, 1]
    assert counters.pages_fetched == 6  # 2 per anchor × 3
    assert counters.lots_fetched == 3  # 1 per anchor
    assert counters.anchors_done == 3


# ── Mid-sweep cancel stops further anchors ──────────────────────────────────


async def test_cancel_midway_stops_remaining_anchors(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    fetch_calls: list[str] = []

    async def fake_fetch(self: YandexRealtyScraper, lat, lon, radius_m=1000, page=0):
        fetch_calls.append(f"{lat}")
        return [_fake_lot(f"{lat}-{page}")]

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(
        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
    )
    # is_cancelled flips True on the 2nd check (before anchor #2) → only anchor #1 runs.
    fake = _FakeRuns(cancel_at_anchor=2)
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=object(), run_id=3, anchors=TEST_ANCHORS, pages_per_anchor=1,
    )
    # Only the first anchor fetched before cancel.
    assert len(fetch_calls) == 1
    assert counters.anchors_done == 1
    # Cancel path returns early: no mark_done, no ban/fail.
    assert fake.done is None
    assert fake.failed is None
    assert fake.banned is None


# ── Per-anchor exception is isolated (single failure → continue) ────────────


async def test_single_anchor_failure_is_isolated(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Anchor #2 raises → anchors #1 and #3 still processed; sweep completes."""
    processed: list[str] = []

    async def fake_fetch(self: YandexRealtyScraper, lat, lon, radius_m=1000, page=0):
        if name_of(lat) == "A2":
            raise RuntimeError("boom on anchor 2")
        processed.append(name_of(lat))
        return [_fake_lot(f"{lat}-{page}")]

    def name_of(lat: float) -> str:
        for a_lat, _lon, nm in TEST_ANCHORS:
            if a_lat == lat:
                return nm
        return "?"

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(
        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
    )
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=object(), run_id=4, anchors=TEST_ANCHORS, pages_per_anchor=1,
    )
    # A1 and A3 processed; A2 raised and was skipped.
    assert processed == ["A1", "A3"]
    assert counters.errors_count == 1
    assert counters.anchors_done == 3  # loop advanced past all 3 anchors
    assert counters.lots_fetched == 2  # only A1 + A3 contributed lots
    # Not an abort — sweep marked done despite the isolated failure.
    assert fake.done is not None
    assert fake.banned is None


# ── Defensive abort: N consecutive failures → mark_banned, stop early ───────


async def test_consecutive_failures_abort_sweep(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """All anchors raise → after MAX_CONSECUTIVE_FAILURES the sweep aborts + bans."""
    attempts: list[str] = []

    async def fake_fetch(self: YandexRealtyScraper, lat, lon, radius_m=1000, page=0):
        attempts.append(f"{lat}")
        raise RuntimeError("yandex blocked")

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(
        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
    )
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    # 5 anchors so the 3-consecutive-failure threshold trips before the list ends.
    anchors = [
        *TEST_ANCHORS,
        (56.7700, 60.5500, "A4"),
        (56.8650, 60.6200, "A5"),
    ]
    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=object(), run_id=5, anchors=anchors, pages_per_anchor=1,
    )
    # Abort exactly at the threshold (3 consecutive failures), not all 5 anchors.
    assert len(attempts) == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
    assert counters.errors_count == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
    assert counters.anchors_done == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
    assert fake.banned is not None
    assert "consecutive" in fake.banned[0]
    assert fake.done is None  # aborted, never marked done


async def test_non_consecutive_failures_do_not_abort(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A success between two failures resets the consecutive counter → no abort."""
    def name_of(lat: float, anchors: list[tuple[float, float, str]]) -> str:
        for a_lat, _lon, nm in anchors:
            if a_lat == lat:
                return nm
        return "?"

    anchors = [
        *TEST_ANCHORS,
        (56.7700, 60.5500, "A4"),
        (56.8650, 60.6200, "A5"),
    ]
    # Fail A1, succeed A2, fail A3, succeed A4, fail A5 → never 3-in-a-row.
    failing = {"A1", "A3", "A5"}

    async def fake_fetch(self: YandexRealtyScraper, lat, lon, radius_m=1000, page=0):
        if name_of(lat, anchors) in failing:
            raise RuntimeError("transient")
        return [_fake_lot(f"{lat}-{page}")]

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(
        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
    )
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=object(), run_id=6, anchors=anchors, pages_per_anchor=1,
    )
    assert counters.errors_count == 3  # A1, A3, A5
    assert counters.anchors_done == 5  # walked the whole list, no early abort
    assert counters.lots_fetched == 2  # A2 + A4 contributed
    assert fake.banned is None
    assert fake.done is not None