gendesign/tradein-mvp/backend/tests/test_n1_city_sweep.py

"""Offline unit tests for run_n1_city_sweep (#575).

Pure & fast: NO live network, NO DB. Mirrors test_yandex_city_sweep — monkeypatch
N1Scraper lifecycle + fetch_around, scrape_pipeline.save_listings, scrape_runs.*.
Asserts orchestration semantics only (live N1 behavior verified by post-deploy QA).

NB: run_n1_city_sweep пагинирует page=1..pages_per_anchor (1-based), в отличие от
yandex (0-based) — тесты учитывают это.
"""

import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from typing import Any

import pytest

from app.services import scrape_pipeline
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.n1 import N1Scraper

TEST_ANCHORS: list[tuple[float, float, str]] = [
    (56.8400, 60.6050, "A1"),
    (56.7950, 60.5300, "A2"),
    (56.8970, 60.6100, "A3"),
]


def _fake_lot(offer_id: str) -> ScrapedLot:
    return ScrapedLot(
        source="n1",
        source_url=f"https://ekb.n1.ru/view/{offer_id}/",
        source_id=offer_id,
        address="Екатеринбург, улица Тестовая, 1",
        price_rub=5_000_000,
        area_m2=50.0,
        rooms=2,
    )


class _FakeRuns:
    def __init__(self, *, cancel_at_anchor: int | None = None) -> None:
        self.cancel_at_anchor = cancel_at_anchor
        self.is_cancelled_calls = 0
        self.heartbeats: list[dict[str, int]] = []
        self.done: dict[str, int] | None = None
        self.failed: tuple[str, dict[str, int]] | None = None
        self.banned: tuple[str, dict[str, int]] | None = None

    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
        self.is_cancelled_calls += 1
        if self.cancel_at_anchor is None:
            return False
        return self.is_cancelled_calls >= self.cancel_at_anchor

    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.heartbeats.append(dict(counters))

    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.done = dict(counters)

    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.failed = (error, dict(counters))

    def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.banned = (error, dict(counters))


@pytest.fixture(autouse=True)
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
    async def _instant(_secs: float) -> None:
        return None

    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)


@pytest.fixture(autouse=True)
def _stub_scraper_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
    """Neutralize __init__/__aenter__/__aexit__ — no get_scraper_delay DB, no curl_cffi."""

    def _init(self: N1Scraper) -> None:
        self.request_delay_sec = 0.0
        self._cffi = None

    async def _aenter(self: N1Scraper) -> N1Scraper:
        return self

    async def _aexit(self: N1Scraper, *_args: Any) -> None:
        return None

    monkeypatch.setattr(N1Scraper, "__init__", _init)
    monkeypatch.setattr(N1Scraper, "__aenter__", _aenter)
    monkeypatch.setattr(N1Scraper, "__aexit__", _aexit)


def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)


async def test_sweep_iterates_all_anchors_and_aggregates(monkeypatch: pytest.MonkeyPatch) -> None:
    fetch_calls: list[tuple[float, float, int]] = []
    save_calls: list[int] = []

    async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1):
        fetch_calls.append((lat, lon, page))
        return [_fake_lot(f"{lat}-{page}-{i}") for i in range(2)]

    def fake_save(_db, lots, *, run_id=None):
        save_calls.append(len(lots))
        return len(lots), 0

    monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_n1_city_sweep(
        db=object(), run_id=1, anchors=TEST_ANCHORS, pages_per_anchor=2, request_delay_sec=0.0
    )

    assert len(fetch_calls) == len(TEST_ANCHORS) * 2
    assert [p for _, _, p in fetch_calls[:2]] == [1, 2]  # 1-based pagination
    assert len(save_calls) == len(TEST_ANCHORS)
    assert counters.anchors_done == len(TEST_ANCHORS)
    assert counters.pages_fetched == len(TEST_ANCHORS) * 2
    assert counters.lots_fetched == 12
    assert counters.lots_inserted == 12
    assert counters.errors_count == 0
    assert fake.done is not None and fake.done["lots_fetched"] == 12
    assert fake.failed is None and fake.banned is None


async def test_save_listings_receives_run_id(monkeypatch: pytest.MonkeyPatch) -> None:
    seen: list[int | None] = []

    async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1):
        return [_fake_lot(f"{lat}-{page}")]

    def fake_save(_db, lots, *, run_id=None):
        seen.append(run_id)
        return len(lots), 0

    monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
    _install_runs(monkeypatch, _FakeRuns())

    await scrape_pipeline.run_n1_city_sweep(
        db=object(), run_id=777, anchors=TEST_ANCHORS, pages_per_anchor=1
    )
    assert seen == [777, 777, 777]


async def test_empty_page_stops_pagination(monkeypatch: pytest.MonkeyPatch) -> None:
    pages_seen: list[int] = []

    async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1):
        pages_seen.append(page)
        if page == 1:
            return [_fake_lot(f"{lat}-1")]
        return []  # page 2 empty → stop

    monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(
        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
    )
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_n1_city_sweep(
        db=object(), run_id=2, anchors=TEST_ANCHORS, pages_per_anchor=5
    )
    assert pages_seen == [1, 2, 1, 2, 1, 2]
    assert counters.pages_fetched == 6
    assert counters.lots_fetched == 3
    assert counters.anchors_done == 3


async def test_cancel_midway_stops_remaining(monkeypatch: pytest.MonkeyPatch) -> None:
    fetch_calls: list[str] = []

    async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1):
        fetch_calls.append(f"{lat}")
        return [_fake_lot(f"{lat}-{page}")]

    monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(
        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
    )
    fake = _FakeRuns(cancel_at_anchor=2)
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_n1_city_sweep(
        db=object(), run_id=3, anchors=TEST_ANCHORS, pages_per_anchor=1
    )
    assert len(fetch_calls) == 1
    assert counters.anchors_done == 1
    assert fake.done is None and fake.banned is None


async def test_consecutive_failures_abort_sweep(monkeypatch: pytest.MonkeyPatch) -> None:
    attempts: list[str] = []

    async def fake_fetch(self, lat, lon, radius_m=1000, rooms=None, page=1):
        attempts.append(f"{lat}")
        raise RuntimeError("n1 blocked")

    monkeypatch.setattr(N1Scraper, "fetch_around", fake_fetch)
    monkeypatch.setattr(
        scrape_pipeline, "save_listings", lambda _db, lots, *, run_id=None: (len(lots), 0)
    )
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    anchors = [*TEST_ANCHORS, (56.77, 60.55, "A4"), (56.865, 60.62, "A5")]
    counters = await scrape_pipeline.run_n1_city_sweep(
        db=object(), run_id=5, anchors=anchors, pages_per_anchor=1
    )
    n = scrape_pipeline.N1_SWEEP_MAX_CONSECUTIVE_FAILURES
    assert len(attempts) == n
    assert counters.errors_count == n
    assert counters.anchors_done == n
    assert fake.banned is not None and "consecutive" in fake.banned[0]
    assert fake.done is None