gendesign/tradein-mvp/backend/tests/test_anchor_watchdog.py

"""Unit tests for per-anchor watchdog timeout in city sweeps (#880).

Verifies that ANCHOR_TIMEOUT_SEC is applied to all three sweeps (avito / cian / yandex):
  (a) A hung anchor (asyncio.sleep(large)) doesn't freeze the whole sweep.
  (b) TimeoutError branch: errors_count incremented, anchor logged as timed-out.
  (c) Sweep proceeds to the next anchor after the timeout.
  (d) mark_done is called when the sweep finishes all anchors.

All tests are network-free — they monkeypatch the inner scrape call.
"""

from __future__ import annotations

import asyncio
import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from typing import Any
from unittest.mock import AsyncMock, MagicMock

import pytest

from app.services import scrape_pipeline
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.cian import CianScraper
from app.services.scrapers.yandex_realty import YandexRealtyScraper

# Two anchors: first hangs (causes timeout), second succeeds.
TWO_ANCHORS: list[tuple[float, float, str]] = [
    (56.8400, 60.6050, "Hang"),
    (56.7950, 60.5300, "OK"),
]


# ── Shared fake scrape_runs ─────────────────────────────────────────────────


class _FakeRuns:
    def __init__(self) -> None:
        self.heartbeats: list[dict[str, int]] = []
        self.done: dict[str, int] | None = None
        self.failed: tuple[str, dict[str, int]] | None = None
        self.banned: tuple[str, dict[str, int]] | None = None

    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
        return False

    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.heartbeats.append(dict(counters))

    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.done = dict(counters)

    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.failed = (error, dict(counters))

    def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.banned = (error, dict(counters))


def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)


def _fake_lot(source: str, offer_id: str) -> ScrapedLot:
    return ScrapedLot(
        source=source,
        source_url=f"https://example.ru/{offer_id}/",
        source_id=offer_id,
        address="Екатеринбург, улица Тестовая, 1",
        price_rub=4_000_000,
        area_m2=42.0,
        rooms=1,
    )


# ── CianScraper lifecycle stub ──────────────────────────────────────────────


@pytest.fixture(autouse=True)
def _stub_cian_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
    def _init(self: CianScraper) -> None:
        self.request_delay_sec = 0.0
        self._cffi = None

    async def _aenter(self: CianScraper) -> CianScraper:
        return self

    async def _aexit(self: CianScraper, *_args: Any) -> None:
        return None

    monkeypatch.setattr(CianScraper, "__init__", _init)
    monkeypatch.setattr(CianScraper, "__aenter__", _aenter)
    monkeypatch.setattr(CianScraper, "__aexit__", _aexit)


# ── YandexRealtyScraper lifecycle stub ─────────────────────────────────────


@pytest.fixture(autouse=True)
def _stub_yandex_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
    def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
        self.city = city
        self.request_delay_sec = 0.0
        self._cffi_session = None
        self._cookies = {}

    async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
        return self

    async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
        return None

    monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
    monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
    monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)


# ── ANCHOR_TIMEOUT_SEC is exported ─────────────────────────────────────────


def test_anchor_timeout_sec_exported() -> None:
    from app.services.scrape_pipeline import ANCHOR_TIMEOUT_SEC

    assert isinstance(ANCHOR_TIMEOUT_SEC, int)
    # Reasonable sane range: 60s–600s
    assert (
        60 <= ANCHOR_TIMEOUT_SEC <= 600
    ), f"ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC} out of sane range"


# ── Avito sweep watchdog ────────────────────────────────────────────────────


async def test_avito_sweep_anchor_timeout_continues(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
    call_count = 0

    # Monkeypatch asyncio.wait_for so we can inject TimeoutError on first call only,
    # without actually waiting 240s in CI.
    original_wait_for = asyncio.wait_for
    wf_call = 0

    async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
        nonlocal wf_call
        wf_call += 1
        if wf_call == 1:
            # Simulate the hung anchor: cancel the coro (mimic wait_for behaviour)
            # and raise TimeoutError.
            try:
                coro.close()
            except Exception:
                pass
            raise TimeoutError
        return await original_wait_for(coro, timeout=timeout)

    monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())

    # Stub run_avito_pipeline so the second anchor returns 3 lots quickly.
    async def fake_pipeline(db: Any, *, lat: float, **_kw: Any) -> Any:
        nonlocal call_count
        call_count += 1
        from app.services.scrape_pipeline import PipelineCounters, PipelineResult

        cnt = PipelineCounters(lots_fetched=3, lots_inserted=3)
        return PipelineResult(lat, 0.0, 1500, cnt, False, 0)

    monkeypatch.setattr(scrape_pipeline, "run_avito_pipeline", fake_pipeline)

    # Stub shared AsyncSession (used by avito sweep).
    fake_session = AsyncMock()
    fake_session.__aenter__ = AsyncMock(return_value=fake_session)
    fake_session.__aexit__ = AsyncMock(return_value=None)
    import curl_cffi.requests as _cffi_mod

    monkeypatch.setattr(_cffi_mod, "AsyncSession", lambda **_kw: fake_session)

    # Stub IMV phase (not relevant to this test).
    async def fake_imv_batch(*_a: Any, **_kw: Any) -> Any:
        result = MagicMock()
        result.checked = 0
        result.saved = 0
        result.errors = 0
        return result

    monkeypatch.setattr(
        "app.services.house_imv_backfill.process_houses_imv_batch",
        fake_imv_batch,
    )

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_avito_city_sweep(
        db=MagicMock(),
        run_id=1,
        anchors=TWO_ANCHORS,
        enrich_houses=False,
        detail_top_n=0,
        enrich_imv=False,
        request_delay_sec=0.0,
    )

    # (a) sweep completes — doesn't hang
    # (b) first anchor timed out → errors_count = 1
    assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
    # (c) second anchor reached: lots_fetched > 0
    assert counters.lots_fetched > 0, "второй anchor должен был выполниться"
    # (d) mark_done вызван (sweep завершился)
    assert fake.done is not None, "mark_done должен быть вызван"
    assert fake.failed is None, "mark_failed не должен быть вызван"
    assert counters.anchors_done == len(TWO_ANCHORS)


# ── Cian sweep watchdog ─────────────────────────────────────────────────────


async def test_cian_sweep_anchor_timeout_continues(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
    serp_calls: list[str] = []
    original_wait_for = asyncio.wait_for
    wf_call = 0

    async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
        nonlocal wf_call
        wf_call += 1
        if wf_call == 1:
            try:
                coro.close()
            except Exception:
                pass
            raise TimeoutError
        return await original_wait_for(coro, timeout=timeout)

    monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())

    async def fake_fetch_multi(
        self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
    ) -> list[ScrapedLot]:
        serp_calls.append(f"{lat:.4f}")
        return [_fake_lot("cian", f"{lat:.4f}-1")]

    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(
        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
    )
    monkeypatch.setattr(
        "app.services.scrapers.cian_detail.fetch_detail",
        AsyncMock(return_value=None),
    )

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_cian_city_sweep(
        db=MagicMock(),
        run_id=2,
        anchors=TWO_ANCHORS,
        detail_top_n=0,
        enrich_houses=False,
        request_delay_sec=0.0,
    )

    # (a) sweep completes (doesn't hang at anchor 1)
    # (b) errors_count >= 1 (timeout counted)
    assert counters.errors_count >= 1
    # (c) second anchor was reached (SERP called once — first anchor timed out before SERP)
    # wf_call==2 means the second wait_for ran → second anchor's coro executed.
    assert wf_call == 2, f"wait_for should be called once per anchor; got {wf_call}"
    # (d) mark_done called
    assert fake.done is not None
    assert fake.failed is None
    assert counters.anchors_done == len(TWO_ANCHORS)


# ── Yandex sweep watchdog ───────────────────────────────────────────────────


async def test_yandex_sweep_anchor_timeout_continues(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Первый anchor виснет → таймаут; второй anchor успешен; mark_done вызывается."""
    serp_calls: list[str] = []
    original_wait_for = asyncio.wait_for
    wf_call = 0

    async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
        nonlocal wf_call
        wf_call += 1
        if wf_call == 1:
            try:
                coro.close()
            except Exception:
                pass
            raise TimeoutError
        return await original_wait_for(coro, timeout=timeout)

    monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1500,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        serp_calls.append(f"{lat:.4f}")
        return [_fake_lot("yandex", f"{lat:.4f}-1")]

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(
        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
    )

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=3,
        anchors=TWO_ANCHORS,
        pages_per_anchor=1,
        enrich_address=False,
        request_delay_sec=0.0,
    )

    # (a) sweep completes
    # (b) timeout counted
    assert counters.errors_count >= 1
    # (c) second anchor reached
    assert wf_call == 2
    # (d) mark_done called
    assert fake.done is not None
    assert fake.failed is None
    assert counters.anchors_done == len(TWO_ANCHORS)


# ── All three: timeout increments errors_count and is logged, sweep reaches mark_done ──


async def test_cian_timeout_increments_counter_and_marks_done(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Детальная проверка: timeout anchor → errors_count+1, sweep продолжается, mark_done.

    Использует fake_wait_for (зеркало первых трёх тестов): первый вызов → TimeoutError
    (имитирует зависший anchor), второй вызов → реальный wait_for (нормальный anchor).
    Проверяет что:
      - errors_count увеличивается
      - второй anchor выполнился (SERP вызван)
      - mark_done вызван (sweep завершился)
      - mark_banned НЕ вызван (одиночный таймаут < порога consecutive failures)
    """
    original_wait_for = asyncio.wait_for
    wf_call = 0

    async def fake_wait_for(coro: Any, *, timeout: float) -> Any:
        nonlocal wf_call
        wf_call += 1
        if wf_call == 1:
            try:
                coro.close()
            except Exception:
                pass
            raise TimeoutError
        return await original_wait_for(coro, timeout=timeout)

    monkeypatch.setattr(scrape_pipeline.asyncio, "wait_for", fake_wait_for)
    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", AsyncMock())

    serp_calls: list[str] = []

    async def fake_fetch(
        self: CianScraper, lat: float, lon: float, r: int, pages: int = 3
    ) -> list[ScrapedLot]:
        serp_calls.append(f"{lat:.4f}")
        return [_fake_lot("cian", f"{lat:.4f}-ok")]

    monkeypatch.setattr(CianScraper, "fetch_around_multi_room", fake_fetch)
    monkeypatch.setattr(
        scrape_pipeline,
        "save_listings",
        lambda _db, lots, *, run_id=None: (len(lots), 0),
    )
    monkeypatch.setattr(
        "app.services.scrapers.cian_detail.fetch_detail",
        AsyncMock(return_value=None),
    )

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_cian_city_sweep(
        db=MagicMock(),
        run_id=10,
        anchors=TWO_ANCHORS,
        detail_top_n=0,
        enrich_houses=False,
        request_delay_sec=0.0,
    )

    # (b) timeout branch: errors_count incremented
    assert counters.errors_count >= 1, "timeout должен увеличить errors_count"
    # (c) second anchor reached (wait_for was called twice — once per anchor)
    assert wf_call == 2, f"wait_for должен быть вызван для каждого anchor; got {wf_call}"
    # (d) sweep reached mark_done
    assert fake.done is not None, "mark_done должен быть вызван"
    assert fake.banned is None, "mark_banned не должен быть вызван для одиночного таймаута"
    assert counters.anchors_done == len(TWO_ANCHORS)