gendesign/tradein-mvp/backend/tests/test_yandex_city_sweep.py

"""Offline unit tests for run_yandex_city_sweep (SERP+address-enrich, #861).

Pure & fast: NO live network, NO DB. We monkeypatch:
  - YandexRealtyScraper.__init__ / __aenter__ / __aexit__ — avoid get_scraper_delay
    DB read + real curl_cffi session creation.
  - YandexRealtyScraper.fetch_around_multi_room — return a fixed list of fake ScrapedLot.
  - scrape_pipeline.save_listings — counter/no-op (no real listings table).
  - scrape_runs.* (is_cancelled / update_heartbeat / mark_*) — in-memory fakes.
  - curl_cffi.requests.AsyncSession — fake for address-enrich HTTP calls.

Asserts orchestration semantics:
  - SERP → save (all anchors, counters aggregated)
  - address-enrich фаза вызывается если enrich_address=True и есть source_ids
  - enrich_address=False → адрес-фаза пропускается
  - cancel → останавливает sweep
  - N consecutive SERP failures → mark_banned + abort
  - per-item address-enrich error не валит sweep
  - mark_done вызывается при нормальном завершении
"""

from __future__ import annotations

import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from typing import Any
from unittest.mock import AsyncMock, MagicMock

import pytest

from app.services import scrape_pipeline
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.yandex_realty import YandexRealtyScraper

# 3 anchors keep tests fast and let us probe cancel/fail semantics.
TEST_ANCHORS: list[tuple[float, float, str]] = [
    (56.8400, 60.6050, "A1"),
    (56.7950, 60.5300, "A2"),
    (56.8970, 60.6100, "A3"),
]


def _fake_lot(offer_id: str) -> ScrapedLot:
    """Minimal valid ScrapedLot (source/source_url required, price_rub > 0)."""
    return ScrapedLot(
        source="yandex",
        source_url=f"https://realty.yandex.ru/offer/{offer_id}/",
        source_id=offer_id,
        address="Екатеринбург, улица Тестовая",
        price_rub=5_000_000,
        area_m2=50.0,
        rooms=2,
    )


class _FakeRuns:
    """In-memory stand-in for app.services.scrape_runs (no DB)."""

    def __init__(self, *, cancel_at_anchor: int | None = None) -> None:
        # cancel_at_anchor: 1-based anchor index at which is_cancelled flips True.
        self.cancel_at_anchor = cancel_at_anchor
        self.is_cancelled_calls = 0
        self.heartbeats: list[dict[str, int]] = []
        self.done: dict[str, int] | None = None
        self.failed: tuple[str, dict[str, int]] | None = None
        self.banned: tuple[str, dict[str, int]] | None = None

    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
        self.is_cancelled_calls += 1
        if self.cancel_at_anchor is None:
            return False
        return self.is_cancelled_calls >= self.cancel_at_anchor

    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.heartbeats.append(dict(counters))

    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.done = dict(counters)

    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.failed = (error, dict(counters))

    def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.banned = (error, dict(counters))


@pytest.fixture(autouse=True)
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
    """Kill inter-anchor asyncio.sleep so tests are instant."""

    async def _instant(_secs: float) -> None:
        return None

    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)


@pytest.fixture(autouse=True)
def _stub_scraper_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
    """Neutralize __init__/__aenter__/__aexit__ — no get_scraper_delay DB, no curl_cffi."""

    def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
        self.city = city
        self.request_delay_sec = 0.0
        self._cffi_session = None
        self._cookies = {}

    async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
        return self

    async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
        return None

    monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
    monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
    monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)


def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)


def _make_db_mock(source_ids: list[str]) -> MagicMock:
    """DB mock для address-enrich фазы: возвращает строки без номера дома."""
    mock_db = MagicMock()
    rows = [
        {
            "id": i + 1,
            "source_id": sid,
            "address": "Екатеринбург, улица Тестовая",
            "source_url": f"https://realty.yandex.ru/offer/{sid}/",
        }
        for i, sid in enumerate(source_ids)
    ]
    mock_db.execute.return_value.mappings.return_value.all.return_value = rows
    return mock_db


def _fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
    return len(lots), 0


# ── YandexCitySweepCounters ─────────────────────────────────────────────────


def test_counters_defaults() -> None:
    from app.services.scrape_pipeline import YandexCitySweepCounters

    c = YandexCitySweepCounters()
    assert c.anchors_total == 0
    assert c.lots_fetched == 0
    assert c.combos_skipped == 0
    assert c.address_attempted == 0
    assert c.address_enriched == 0
    assert c.address_failed == 0
    assert c.errors_count == 0


def test_counters_to_dict_all_keys() -> None:
    from dataclasses import fields

    from app.services.scrape_pipeline import YandexCitySweepCounters

    c = YandexCitySweepCounters()
    d = c.to_dict()
    expected = {f.name for f in fields(c)}
    assert set(d.keys()) == expected


# ── Happy path: all anchors iterated, counters aggregated ───────────────────


async def test_sweep_iterates_all_anchors_and_aggregates(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """fetch_around_multi_room вызывается для каждого anchor, counters агрегируются.

    FIX: fake_fetch_multi вызывает on_combo (инкрементальный save per-combo) —
    имитирует реальное поведение scraper'а. save_listings должен вызываться
    on_combo'м, а не одним батчем в конце anchor'а.
    """
    serp_calls: list[str] = []
    save_calls: list[int] = []

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        on_combo: Any = None,
        **_: Any,
    ) -> list[ScrapedLot]:
        serp_calls.append(f"{lat:.4f}")
        lots = [_fake_lot(f"{lat}-{i}") for i in range(3)]
        # Имитируем один combo — вызываем on_combo если предоставлен.
        if on_combo is not None:
            on_combo(f"vtorichka/combo_{lat:.4f}", lots)
        return lots

    def fake_save(
        _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
    ) -> tuple[int, int]:
        save_calls.append(len(lots))
        return len(lots), 0

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=1,
        anchors=TEST_ANCHORS,
        pages_per_anchor=2,
        request_delay_sec=0.0,
        enrich_address=False,
    )

    assert len(serp_calls) == len(TEST_ANCHORS)
    # save вызывается инкрементально через on_combo — по одному разу на anchor
    # (fake имитирует один combo per anchor).
    assert len(save_calls) == len(TEST_ANCHORS)
    assert counters.anchors_total == len(TEST_ANCHORS)
    assert counters.anchors_done == len(TEST_ANCHORS)
    assert counters.lots_fetched == len(TEST_ANCHORS) * 3
    assert counters.lots_inserted == len(TEST_ANCHORS) * 3
    assert counters.lots_updated == 0
    assert counters.errors_count == 0
    assert counters.address_attempted == 0  # enrich_address=False
    assert fake.done is not None
    assert fake.done["lots_fetched"] == len(TEST_ANCHORS) * 3
    assert fake.failed is None
    assert fake.banned is None


# ── enrich_address=False пропускает address-фазу ────────────────────────────


async def test_enrich_address_false_skips_address_phase(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """enrich_address=False → address-enrich фаза не выполняется."""
    address_http_calls: list[str] = []

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        return [_fake_lot(f"{lat}-0")]

    async def fake_cffi_get(url: str, **_: Any) -> Any:
        address_http_calls.append(url)
        return MagicMock(status_code=200, text="<title>test</title>")

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=2,
        anchors=TEST_ANCHORS[:1],
        enrich_address=False,
        request_delay_sec=0.0,
    )

    assert len(address_http_calls) == 0
    assert counters.address_attempted == 0
    assert counters.address_enriched == 0
    assert fake.done is not None


# ── address-enrich фаза: счётчики заполняются ───────────────────────────────


async def test_address_enrich_fills_counters(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """address-enrich вызывается для листингов без номера дома; counters заполняются."""
    # Fake SERP возвращает 2 листинга с source_id
    lots = [_fake_lot("id1"), _fake_lot("id2")]

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        on_combo: Any = None,
        **_: Any,
    ) -> list[ScrapedLot]:
        # Вызываем on_combo чтобы anchor_lots накопился (инкрементальный save).
        if on_combo is not None:
            on_combo("vtorichka/combo_enrich_test", lots)
        return lots

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)

    # DB mock: возвращает 2 строки для address-enrich запроса
    mock_db = _make_db_mock(["id1", "id2"])

    # Title с полным адресом — должен быть извлечён через _extract_address_from_title
    fake_response = MagicMock()
    fake_response.status_code = 200
    fake_response.text = (
        "<title>Продажа квартиры — Екатеринбург, улица Тестовая, 36 — id 123 "
        "на Яндекс.Недвижимости</title>"
    )

    # Мок curl_cffi AsyncSession как context manager
    mock_session = AsyncMock()
    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
    mock_session.__aexit__ = AsyncMock(return_value=None)
    mock_session.get = AsyncMock(return_value=fake_response)

    import curl_cffi.requests as cffi_mod

    monkeypatch.setattr(cffi_mod, "AsyncSession", lambda **_kw: mock_session)

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=mock_db,
        run_id=3,
        anchors=TEST_ANCHORS[:1],
        enrich_address=True,
        request_delay_sec=0.0,
    )

    assert counters.address_attempted == 2
    assert counters.address_enriched == 2
    assert counters.address_failed == 0
    assert fake.done is not None


# ── address-enrich: per-item HTTP error не валит sweep ──────────────────────


async def test_address_enrich_http_error_does_not_abort(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """HTTP 403 при address-enrich → address_failed++, sweep продолжается."""

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        on_combo: Any = None,
        **_: Any,
    ) -> list[ScrapedLot]:
        lots = [_fake_lot("id-err")]
        if on_combo is not None:
            on_combo("vtorichka/combo_http_err", lots)
        return lots

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)

    mock_db = _make_db_mock(["id-err"])

    fake_response = MagicMock()
    fake_response.status_code = 403
    fake_response.text = ""

    mock_session = AsyncMock()
    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
    mock_session.__aexit__ = AsyncMock(return_value=None)
    mock_session.get = AsyncMock(return_value=fake_response)

    import curl_cffi.requests as cffi_mod

    monkeypatch.setattr(cffi_mod, "AsyncSession", lambda **_kw: mock_session)

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=mock_db,
        run_id=4,
        anchors=TEST_ANCHORS[:1],
        enrich_address=True,
        request_delay_sec=0.0,
    )

    assert counters.address_attempted == 1
    assert counters.address_failed == 1
    assert counters.address_enriched == 0
    assert fake.done is not None  # sweep завершился несмотря на ошибку


# ── Mid-sweep cancel stops further anchors ──────────────────────────────────


async def test_cancel_midway_stops_remaining_anchors(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    serp_calls: list[str] = []

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        serp_calls.append(f"{lat}")
        return [_fake_lot(f"{lat}-0")]

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    # is_cancelled flips True on the 2nd check (before anchor #2) → only anchor #1 runs.
    fake = _FakeRuns(cancel_at_anchor=2)
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=5,
        anchors=TEST_ANCHORS,
        pages_per_anchor=1,
        enrich_address=False,
        request_delay_sec=0.0,
    )
    # Only the first anchor fetched before cancel.
    assert len(serp_calls) == 1
    assert counters.anchors_done == 1
    # Cancel path returns early: no mark_done, no ban/fail.
    assert fake.done is None
    assert fake.failed is None
    assert fake.banned is None


# ── Consecutive failures → mark_banned + abort ──────────────────────────────


async def test_consecutive_failures_abort_sweep(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """All anchors raise → after MAX_CONSECUTIVE_FAILURES the sweep aborts + bans."""
    attempts: list[str] = []

    async def fake_fetch_fail(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        attempts.append(f"{lat}")
        raise RuntimeError("yandex blocked")

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_fail)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    # 5 anchors so the 3-consecutive-failure threshold trips before the list ends.
    anchors = [
        *TEST_ANCHORS,
        (56.7700, 60.5500, "A4"),
        (56.8650, 60.6200, "A5"),
    ]
    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=6,
        anchors=anchors,
        enrich_address=False,
        request_delay_sec=0.0,
    )
    # Abort exactly at the threshold (3 consecutive failures).
    assert len(attempts) == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
    assert counters.errors_count == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
    assert counters.anchors_done == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
    assert fake.banned is not None
    assert "consecutive" in fake.banned[0]
    assert fake.done is None  # aborted, never marked done


# ── Non-consecutive failures do not abort ───────────────────────────────────


async def test_non_consecutive_failures_do_not_abort(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A success between two failures resets the consecutive counter → no abort."""

    def _name_of(lat: float, anchors: list[tuple[float, float, str]]) -> str:
        for a_lat, _lon, nm in anchors:
            if a_lat == lat:
                return nm
        return "?"

    anchors = [
        *TEST_ANCHORS,
        (56.7700, 60.5500, "A4"),
        (56.8650, 60.6200, "A5"),
    ]
    failing = {"A1", "A3", "A5"}

    async def fake_fetch(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        on_combo: Any = None,
        **_: Any,
    ) -> list[ScrapedLot]:
        if _name_of(lat, anchors) in failing:
            raise RuntimeError("transient")
        lots = [_fake_lot(f"{lat}-0")]
        # Вызываем on_combo для инкрементального save.
        if on_combo is not None:
            on_combo(f"vtorichka/combo_{lat:.4f}", lots)
        return lots

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=7,
        anchors=anchors,
        enrich_address=False,
        request_delay_sec=0.0,
    )
    assert counters.errors_count == 3  # A1, A3, A5
    assert counters.anchors_done == 5  # walked the whole list, no early abort
    assert counters.lots_fetched == 2  # A2 + A4 contributed
    assert fake.banned is None
    assert fake.done is not None


# ── mark_done вызывается даже при пустом SERP ───────────────────────────────


async def test_mark_done_always_called(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    async def fake_fetch_empty(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        return []

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_empty)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=8,
        anchors=TEST_ANCHORS,
        enrich_address=False,
        request_delay_sec=0.0,
    )

    assert fake.done is not None
    assert counters.lots_fetched == 0
    assert counters.anchors_done == len(TEST_ANCHORS)


# ── Admin endpoint tests (offline) ──────────────────────────────────────────


@pytest.fixture
def app_with_admin():
    from fastapi import FastAPI
    from fastapi.testclient import TestClient

    from app.api.v1 import admin as admin_module
    from app.core.db import get_db

    app = FastAPI()
    app.include_router(admin_module.router, prefix="/api/v1/admin")

    def fake_db():
        yield MagicMock()

    app.dependency_overrides[get_db] = fake_db
    return TestClient(app)


def test_yandex_start_endpoint_validates_pages_per_anchor(app_with_admin) -> None:
    """pages_per_anchor=99 превышает max=10 → 422."""
    r = app_with_admin.post(
        "/api/v1/admin/scrape/yandex-city-sweep",
        json={"pages_per_anchor": 99},
    )
    assert r.status_code == 422


def test_yandex_start_endpoint_validates_delay_too_low(app_with_admin) -> None:
    """request_delay_sec=1.0 меньше min=3.0 → 422."""
    r = app_with_admin.post(
        "/api/v1/admin/scrape/yandex-city-sweep",
        json={"request_delay_sec": 1.0},
    )
    assert r.status_code == 422


def test_yandex_start_endpoint_ok(app_with_admin) -> None:
    """Valid request → 200, run_id в ответе."""
    from unittest.mock import patch

    with (
        patch("app.services.scrape_runs.create_run", return_value=55),
        patch("app.services.scrape_pipeline.run_yandex_city_sweep", new_callable=AsyncMock),
    ):
        r = app_with_admin.post(
            "/api/v1/admin/scrape/yandex-city-sweep",
            json={"pages_per_anchor": 2, "enrich_address": True},
        )
    assert r.status_code == 200
    body = r.json()
    assert body["run_id"] == 55
    assert body["status"] == "running"
    assert body["pages_per_anchor"] == 2
    assert body["detail_top_n"] == 0


def test_yandex_cancel_endpoint(app_with_admin) -> None:
    """Cancel endpoint возвращает run_id + cancelled=True."""
    from unittest.mock import patch

    with patch("app.services.scrape_runs.mark_cancelled", return_value=True):
        r = app_with_admin.post("/api/v1/admin/scrape/yandex-city-sweep/99/cancel")
    assert r.status_code == 200
    body = r.json()
    assert body["run_id"] == 99
    assert body["cancelled"] is True
    assert body["ok"] is True


def test_yandex_cancel_endpoint_not_running(app_with_admin) -> None:
    """Cancel на не-running sweep → cancelled=False."""
    from unittest.mock import patch

    with patch("app.services.scrape_runs.mark_cancelled", return_value=False):
        r = app_with_admin.post("/api/v1/admin/scrape/yandex-city-sweep/100/cancel")
    assert r.status_code == 200
    assert r.json()["cancelled"] is False


def test_yandex_list_runs_endpoint(app_with_admin) -> None:
    """List runs endpoint возвращает список."""
    from datetime import UTC, datetime
    from unittest.mock import patch

    fake_rows = [
        {
            "run_id": 20,
            "source": "yandex_city_sweep",
            "status": "done",
            "params": {"pages_per_anchor": 2, "enrich_address": True},
            "counters": {"lots_fetched": 200, "address_enriched": 80},
            "error": None,
            "started_at": datetime(2025, 5, 1, tzinfo=UTC),
            "finished_at": datetime(2025, 5, 1, 1, tzinfo=UTC),
            "heartbeat_at": datetime(2025, 5, 1, 1, tzinfo=UTC),
        }
    ]
    with patch("app.services.scrape_runs.list_recent", return_value=fake_rows):
        r = app_with_admin.get("/api/v1/admin/scrape/yandex-city-sweep/runs")
    assert r.status_code == 200
    body = r.json()
    assert len(body) == 1
    assert body[0]["run_id"] == 20
    assert body[0]["source"] == "yandex_city_sweep"
    assert "2025-05-01" in body[0]["started_at"]


# ── Center-combos wiring: prod (anchors=None) uses EKB center + combos ──────


async def test_prod_mode_uses_center_anchor_with_combos(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """anchors=None (prod mode) → fetch_around_multi_room called with EKB center coords
    AND rooms_list + price_ranges (combos mode), NOT with None/None (legacy mode).
    anchors_total == 1 (single center sweep).
    """
    from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH

    calls: list[dict] = []

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        rooms_list: list | None = None,
        price_ranges: list | None = None,
        on_combo: Any = None,
        **_: Any,
    ) -> list[ScrapedLot]:
        lots = [_fake_lot("center-1"), _fake_lot("center-2")]
        calls.append(
            {
                "lat": lat,
                "lon": lon,
                "radius_m": radius_m,
                "rooms_list": rooms_list,
                "price_ranges": price_ranges,
                "max_pages": max_pages,
            }
        )
        # Имитируем on_combo (инкрементальный save) — один combo.
        if on_combo is not None:
            on_combo("vtorichka/center_combo", lots)
        return lots

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=10,
        # anchors=None → prod mode (EKB center)
        pages_per_anchor=3,
        enrich_address=False,
        request_delay_sec=0.0,
    )

    # Single center anchor
    assert counters.anchors_total == 1
    assert counters.anchors_done == 1

    # fetch_around_multi_room called exactly once
    assert len(calls) == 1
    call = calls[0]

    # EKB center coords
    assert abs(call["lat"] - 56.8400) < 0.0001
    assert abs(call["lon"] - 60.6050) < 0.0001

    # radius covers the whole city (25km default)
    assert call["radius_m"] >= 10000

    # Combos mode: rooms_list and price_ranges are NOT None
    assert call["rooms_list"] is not None
    assert call["price_ranges"] is not None

    # rooms_list covers all ROOM_PATH keys
    assert set(call["rooms_list"]) == set(ROOM_PATH.keys())

    # price_ranges matches DEFAULT_PRICE_RANGES
    assert call["price_ranges"] == DEFAULT_PRICE_RANGES

    # max_pages forwarded from pages_per_anchor
    assert call["max_pages"] == 3

    # lots aggregated correctly via on_combo
    assert counters.lots_fetched == 2
    assert counters.lots_inserted == 2
    assert fake.done is not None
    assert fake.failed is None


async def test_explicit_anchors_still_works(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """anchors=[...] (test/manual override) → fetch called for each anchor, combos mode."""
    calls: list[dict] = []

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        rooms_list: list | None = None,
        price_ranges: list | None = None,
        **_: Any,
    ) -> list[ScrapedLot]:
        calls.append({"lat": lat, "rooms_list": rooms_list, "price_ranges": price_ranges})
        return [_fake_lot(f"{lat}")]

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    two_anchors = [(56.8400, 60.6050, "A"), (56.7950, 60.5300, "B")]
    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=11,
        anchors=two_anchors,
        pages_per_anchor=2,
        enrich_address=False,
        request_delay_sec=0.0,
    )

    # Both anchors iterated
    assert len(calls) == 2
    assert counters.anchors_total == 2
    assert counters.anchors_done == 2

    # Each call uses combos (not None/None)
    for c in calls:
        assert c["rooms_list"] is not None
        assert c["price_ranges"] is not None

    assert fake.done is not None


# ── Watchdog timeout: combos vs explicit-anchor mode ───────────────────────


def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None:
    """В combos/center mode (anchors=None) _sweep_timeout намного > ANCHOR_TIMEOUT_SEC.

    Проверяет что вычисленный таймаут для дефолтных 30 combos × 3 pages > 1500s,
    т.е. не равен ANCHOR_TIMEOUT_SEC=240 и обеспечивает полный прогон sweep'а.
    Не нужны мокапы — только публичные константы и арифметика из модуля.
    """
    from app.services.scrape_pipeline import (
        _YANDEX_ADDRESS_ENRICH_BUDGET_S,
        _YANDEX_COMBOS_PER_FETCH_S,
        ANCHOR_TIMEOUT_SEC,
    )
    from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH

    # Параметры prod-режима
    rooms_list = list(ROOM_PATH.keys())  # 5 комнатностей
    price_ranges = DEFAULT_PRICE_RANGES  # 6 ценовых диапазонов
    max_pages = 3
    request_delay_sec = 9.0  # продакшн default

    num_combos = len(rooms_list) * len(price_ranges)
    assert num_combos == 30, f"Expected 30 combos, got {num_combos}"

    sweep_timeout = max(
        ANCHOR_TIMEOUT_SEC,
        num_combos * max_pages * (request_delay_sec + _YANDEX_COMBOS_PER_FETCH_S)
        + _YANDEX_ADDRESS_ENRICH_BUDGET_S,
    )

    # Должен быть существенно > 1500s (≈ 25 мин), что покрывает полный run.
    assert sweep_timeout > 1500, (
        f"sweep_timeout={sweep_timeout:.0f}s unexpectedly small — "
        f"combos-mode will still timeout mid-sweep"
    )
    # И намного > ANCHOR_TIMEOUT_SEC (240s)
    assert (
        sweep_timeout > ANCHOR_TIMEOUT_SEC * 4
    ), f"sweep_timeout={sweep_timeout:.0f}s should be >> ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC}s"


async def test_explicit_anchor_mode_uses_anchor_timeout_sec(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """anchors=[...] (explicit override) → wait_for используется с ANCHOR_TIMEOUT_SEC.

    Проверяет что в legacy/test режиме (explicit anchors) sweep_timeout не раздут
    до combos-значения — он равен ANCHOR_TIMEOUT_SEC, т.к. каждый anchor маленький.
    """
    import asyncio as _asyncio

    from app.services.scrape_pipeline import ANCHOR_TIMEOUT_SEC

    captured_timeouts: list[float] = []

    _orig_wait_for = _asyncio.wait_for

    async def _capturing_wait_for(coro, *, timeout):  # type: ignore[no-untyped-def]
        captured_timeouts.append(timeout)
        # На самом деле запустить — мокнутый scraper вернёт [] мгновенно.
        return await _orig_wait_for(coro, timeout=timeout)

    monkeypatch.setattr(_asyncio, "wait_for", _capturing_wait_for)

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: object,
    ) -> list[ScrapedLot]:
        return []

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=20,
        anchors=TEST_ANCHORS[:2],  # explicit anchors — legacy mode
        pages_per_anchor=2,
        enrich_address=False,
        request_delay_sec=0.0,
    )

    # В explicit-anchor режиме каждый anchor должен использовать ANCHOR_TIMEOUT_SEC
    assert len(captured_timeouts) == 2
    for t in captured_timeouts:
        assert (
            t == ANCHOR_TIMEOUT_SEC
        ), f"Expected ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC}s, got {t}s"


# ── FIX A: инкрементальный save per-combo ────────────────────────────────────


async def test_incremental_save_called_per_combo(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """save_listings вызывается on_combo per-combo, не одним батчем в конце anchor'а.

    Имитируем 3 combo через on_combo — save_listings должен быть вызван 3 раза
    (по одному на combo) с порциями лотов. Проверяем, что при краше anchor'а
    уже сохранённые combo не теряются (данные в БД до finish anchor'а).
    """
    save_batches: list[list[str]] = []

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        on_combo: Any = None,
        **_: Any,
    ) -> list[ScrapedLot]:
        all_lots = []
        for combo_idx in range(3):
            combo_lots = [_fake_lot(f"{lat}-combo{combo_idx}-lot0")]
            all_lots.extend(combo_lots)
            if on_combo is not None:
                on_combo(f"vtorichka/combo_{combo_idx}", combo_lots)
        return all_lots

    def fake_save(
        _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
    ) -> tuple[int, int]:
        save_batches.append([lot.source_id for lot in lots if lot.source_id])
        return len(lots), 0

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=30,
        anchors=TEST_ANCHORS[:1],
        pages_per_anchor=2,
        request_delay_sec=0.0,
        enrich_address=False,
    )

    # save_listings вызван ровно 3 раза (по одному на combo), не одним батчем.
    assert (
        len(save_batches) == 3
    ), f"Ожидалось 3 save вызова (per-combo), получено {len(save_batches)}: {save_batches}"
    # Каждый batch содержит ровно 1 лот (размер combo).
    for batch in save_batches:
        assert len(batch) == 1, f"Каждый combo-batch должен иметь 1 лот, получено {batch}"

    # Суммарные counters корректны.
    assert counters.lots_fetched == 3
    assert counters.lots_inserted == 3
    assert fake.done is not None

    # heartbeat вызывался после каждого combo (через on_combo).
    assert (
        len(fake.heartbeats) >= 3
    ), f"Ожидалось ≥3 heartbeat обновлений (per-combo), получено {len(fake.heartbeats)}"


# ── FIX B: bounded retry — скипает combo после exhaustion ────────────────────


async def test_bounded_retry_combo_skipped_logged(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """on_combo НЕ вызывается если combo пропущен (все retry page-1 исчерпаны).

    Имитируем sweep с двумя combos: первый скипается (on_combo не вызван),
    второй успешен (on_combo вызван). Проверяем, что save_listings вызван
    только для успешного combo, а не скипнутого.
    """
    save_batches: list[list[str]] = []
    on_combo_calls: list[str] = []

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        on_combo: Any = None,
        **_: Any,
    ) -> list[ScrapedLot]:
        # Combo 0: скипается — on_combo НЕ вызывается (симулирует exhausted retries).
        # Combo 1: успешен — on_combo вызывается с лотом.
        success_lot = _fake_lot(f"{lat}-success")
        if on_combo is not None:
            on_combo_calls.append("combo_1_success")
            on_combo("vtorichka/combo_1", [success_lot])
        return [success_lot]

    def fake_save(
        _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
    ) -> tuple[int, int]:
        save_batches.append([lot.source_id for lot in lots if lot.source_id])
        return len(lots), 0

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=31,
        anchors=TEST_ANCHORS[:1],
        pages_per_anchor=2,
        request_delay_sec=0.0,
        enrich_address=False,
    )

    # Только успешный combo вызвал on_combo и save.
    assert len(on_combo_calls) == 1
    assert len(save_batches) == 1
    assert counters.lots_fetched == 1
    assert counters.lots_inserted == 1
    assert fake.done is not None