"""Offline unit tests for run_yandex_city_sweep (SERP+address-enrich, #861).

Pure & fast: NO live network, NO DB. We monkeypatch:
  - YandexRealtyScraper.__init__ / __aenter__ / __aexit__ — avoid get_scraper_delay
    DB read + real curl_cffi session creation.
  - YandexRealtyScraper.fetch_around_multi_room — return a fixed list of fake ScrapedLot.
  - scrape_pipeline.save_listings — counter/no-op (no real listings table).
  - scrape_runs.* (is_cancelled / update_heartbeat / mark_*) — in-memory fakes.
  - curl_cffi.requests.AsyncSession — fake for address-enrich HTTP calls.

Asserts orchestration semantics:
  - SERP → save (all anchors, counters aggregated)
  - address-enrich фаза вызывается если enrich_address=True и есть source_ids
  - enrich_address=False → адрес-фаза пропускается
  - cancel → останавливает sweep
  - N consecutive SERP failures → mark_banned + abort
  - per-item address-enrich error не валит sweep
  - mark_done вызывается при нормальном завершении
"""

from __future__ import annotations

import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from typing import Any
from unittest.mock import AsyncMock, MagicMock

import pytest

from app.services import scrape_pipeline
from app.services.scrapers.base import ScrapedLot
from app.services.scrapers.yandex_realty import YandexRealtyScraper

# 3 anchors keep tests fast and let us probe cancel/fail semantics.
TEST_ANCHORS: list[tuple[float, float, str]] = [
    (56.8400, 60.6050, "A1"),
    (56.7950, 60.5300, "A2"),
    (56.8970, 60.6100, "A3"),
]


def _fake_lot(offer_id: str) -> ScrapedLot:
    """Minimal valid ScrapedLot (source/source_url required, price_rub > 0)."""
    return ScrapedLot(
        source="yandex",
        source_url=f"https://realty.yandex.ru/offer/{offer_id}/",
        source_id=offer_id,
        address="Екатеринбург, улица Тестовая",
        price_rub=5_000_000,
        area_m2=50.0,
        rooms=2,
    )


class _FakeRuns:
    """In-memory stand-in for app.services.scrape_runs (no DB)."""

    def __init__(self, *, cancel_at_anchor: int | None = None) -> None:
        # cancel_at_anchor: 1-based anchor index at which is_cancelled flips True.
        self.cancel_at_anchor = cancel_at_anchor
        self.is_cancelled_calls = 0
        self.heartbeats: list[dict[str, int]] = []
        self.done: dict[str, int] | None = None
        self.failed: tuple[str, dict[str, int]] | None = None
        self.banned: tuple[str, dict[str, int]] | None = None

    def is_cancelled(self, _db: Any, _run_id: int) -> bool:
        self.is_cancelled_calls += 1
        if self.cancel_at_anchor is None:
            return False
        return self.is_cancelled_calls >= self.cancel_at_anchor

    def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.heartbeats.append(dict(counters))

    def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None:
        self.done = dict(counters)

    def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.failed = (error, dict(counters))

    def mark_banned(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None:
        self.banned = (error, dict(counters))


@pytest.fixture(autouse=True)
def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None:
    """Kill inter-anchor asyncio.sleep so tests are instant."""

    async def _instant(_secs: float) -> None:
        return None

    monkeypatch.setattr(scrape_pipeline.asyncio, "sleep", _instant)


@pytest.fixture(autouse=True)
def _stub_scraper_lifecycle(monkeypatch: pytest.MonkeyPatch) -> None:
    """Neutralize __init__/__aenter__/__aexit__ — no get_scraper_delay DB, no curl_cffi."""

    def _init(self: YandexRealtyScraper, city: str = "ekaterinburg") -> None:
        self.city = city
        self.request_delay_sec = 0.0
        self._cffi_session = None
        self._cookies = {}

    async def _aenter(self: YandexRealtyScraper) -> YandexRealtyScraper:
        return self

    async def _aexit(self: YandexRealtyScraper, *_args: Any) -> None:
        return None

    monkeypatch.setattr(YandexRealtyScraper, "__init__", _init)
    monkeypatch.setattr(YandexRealtyScraper, "__aenter__", _aenter)
    monkeypatch.setattr(YandexRealtyScraper, "__aexit__", _aexit)


def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None:
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "is_cancelled", fake.is_cancelled)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "update_heartbeat", fake.update_heartbeat)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_done", fake.mark_done)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_failed", fake.mark_failed)
    monkeypatch.setattr(scrape_pipeline.scrape_runs, "mark_banned", fake.mark_banned)


def _make_db_mock(source_ids: list[str]) -> MagicMock:
    """DB mock для address-enrich фазы: возвращает строки без номера дома."""
    mock_db = MagicMock()
    rows = [
        {
            "id": i + 1,
            "source_id": sid,
            "address": "Екатеринбург, улица Тестовая",
            "source_url": f"https://realty.yandex.ru/offer/{sid}/",
        }
        for i, sid in enumerate(source_ids)
    ]
    mock_db.execute.return_value.mappings.return_value.all.return_value = rows
    return mock_db


def _fake_save(_db: Any, lots: list, *, run_id: int | None = None) -> tuple[int, int]:
    return len(lots), 0


# ── YandexCitySweepCounters ─────────────────────────────────────────────────


def test_counters_defaults() -> None:
    from app.services.scrape_pipeline import YandexCitySweepCounters

    c = YandexCitySweepCounters()
    assert c.anchors_total == 0
    assert c.lots_fetched == 0
    assert c.address_attempted == 0
    assert c.address_enriched == 0
    assert c.address_failed == 0
    assert c.errors_count == 0


def test_counters_to_dict_all_keys() -> None:
    from dataclasses import fields

    from app.services.scrape_pipeline import YandexCitySweepCounters

    c = YandexCitySweepCounters()
    d = c.to_dict()
    expected = {f.name for f in fields(c)}
    assert set(d.keys()) == expected


# ── Happy path: all anchors iterated, counters aggregated ───────────────────


async def test_sweep_iterates_all_anchors_and_aggregates(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """fetch_around_multi_room вызывается для каждого anchor, counters агрегируются."""
    serp_calls: list[str] = []
    save_calls: list[int] = []

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        serp_calls.append(f"{lat:.4f}")
        return [_fake_lot(f"{lat}-{i}") for i in range(3)]

    def fake_save(
        _db: Any, lots: list[ScrapedLot], *, run_id: int | None = None
    ) -> tuple[int, int]:
        save_calls.append(len(lots))
        return len(lots), 0

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=1,
        anchors=TEST_ANCHORS,
        pages_per_anchor=2,
        request_delay_sec=0.0,
        enrich_address=False,
    )

    assert len(serp_calls) == len(TEST_ANCHORS)
    assert len(save_calls) == len(TEST_ANCHORS)
    assert counters.anchors_total == len(TEST_ANCHORS)
    assert counters.anchors_done == len(TEST_ANCHORS)
    assert counters.lots_fetched == len(TEST_ANCHORS) * 3
    assert counters.lots_inserted == len(TEST_ANCHORS) * 3
    assert counters.lots_updated == 0
    assert counters.errors_count == 0
    assert counters.address_attempted == 0  # enrich_address=False
    assert fake.done is not None
    assert fake.done["lots_fetched"] == len(TEST_ANCHORS) * 3
    assert fake.failed is None
    assert fake.banned is None


# ── enrich_address=False пропускает address-фазу ────────────────────────────


async def test_enrich_address_false_skips_address_phase(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """enrich_address=False → address-enrich фаза не выполняется."""
    address_http_calls: list[str] = []

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        return [_fake_lot(f"{lat}-0")]

    async def fake_cffi_get(url: str, **_: Any) -> Any:
        address_http_calls.append(url)
        return MagicMock(status_code=200, text="<title>test</title>")

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=2,
        anchors=TEST_ANCHORS[:1],
        enrich_address=False,
        request_delay_sec=0.0,
    )

    assert len(address_http_calls) == 0
    assert counters.address_attempted == 0
    assert counters.address_enriched == 0
    assert fake.done is not None


# ── address-enrich фаза: счётчики заполняются ───────────────────────────────


async def test_address_enrich_fills_counters(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """address-enrich вызывается для листингов без номера дома; counters заполняются."""
    # Fake SERP возвращает 2 листинга с source_id
    lots = [_fake_lot("id1"), _fake_lot("id2")]

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        return lots

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)

    # DB mock: возвращает 2 строки для address-enrich запроса
    mock_db = _make_db_mock(["id1", "id2"])

    # Title с полным адресом — должен быть извлечён через _extract_address_from_title
    fake_response = MagicMock()
    fake_response.status_code = 200
    fake_response.text = (
        "<title>Продажа квартиры — Екатеринбург, улица Тестовая, 36 — id 123 "
        "на Яндекс.Недвижимости</title>"
    )

    # Мок curl_cffi AsyncSession как context manager
    mock_session = AsyncMock()
    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
    mock_session.__aexit__ = AsyncMock(return_value=None)
    mock_session.get = AsyncMock(return_value=fake_response)

    import curl_cffi.requests as cffi_mod

    monkeypatch.setattr(cffi_mod, "AsyncSession", lambda **_kw: mock_session)

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=mock_db,
        run_id=3,
        anchors=TEST_ANCHORS[:1],
        enrich_address=True,
        request_delay_sec=0.0,
    )

    assert counters.address_attempted == 2
    assert counters.address_enriched == 2
    assert counters.address_failed == 0
    assert fake.done is not None


# ── address-enrich: per-item HTTP error не валит sweep ──────────────────────


async def test_address_enrich_http_error_does_not_abort(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """HTTP 403 при address-enrich → address_failed++, sweep продолжается."""

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        return [_fake_lot("id-err")]

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)

    mock_db = _make_db_mock(["id-err"])

    fake_response = MagicMock()
    fake_response.status_code = 403
    fake_response.text = ""

    mock_session = AsyncMock()
    mock_session.__aenter__ = AsyncMock(return_value=mock_session)
    mock_session.__aexit__ = AsyncMock(return_value=None)
    mock_session.get = AsyncMock(return_value=fake_response)

    import curl_cffi.requests as cffi_mod

    monkeypatch.setattr(cffi_mod, "AsyncSession", lambda **_kw: mock_session)

    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=mock_db,
        run_id=4,
        anchors=TEST_ANCHORS[:1],
        enrich_address=True,
        request_delay_sec=0.0,
    )

    assert counters.address_attempted == 1
    assert counters.address_failed == 1
    assert counters.address_enriched == 0
    assert fake.done is not None  # sweep завершился несмотря на ошибку


# ── Mid-sweep cancel stops further anchors ──────────────────────────────────


async def test_cancel_midway_stops_remaining_anchors(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    serp_calls: list[str] = []

    async def fake_fetch_multi(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        serp_calls.append(f"{lat}")
        return [_fake_lot(f"{lat}-0")]

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_multi)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    # is_cancelled flips True on the 2nd check (before anchor #2) → only anchor #1 runs.
    fake = _FakeRuns(cancel_at_anchor=2)
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=5,
        anchors=TEST_ANCHORS,
        pages_per_anchor=1,
        enrich_address=False,
        request_delay_sec=0.0,
    )
    # Only the first anchor fetched before cancel.
    assert len(serp_calls) == 1
    assert counters.anchors_done == 1
    # Cancel path returns early: no mark_done, no ban/fail.
    assert fake.done is None
    assert fake.failed is None
    assert fake.banned is None


# ── Consecutive failures → mark_banned + abort ──────────────────────────────


async def test_consecutive_failures_abort_sweep(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """All anchors raise → after MAX_CONSECUTIVE_FAILURES the sweep aborts + bans."""
    attempts: list[str] = []

    async def fake_fetch_fail(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        attempts.append(f"{lat}")
        raise RuntimeError("yandex blocked")

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_fail)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    # 5 anchors so the 3-consecutive-failure threshold trips before the list ends.
    anchors = [
        *TEST_ANCHORS,
        (56.7700, 60.5500, "A4"),
        (56.8650, 60.6200, "A5"),
    ]
    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=6,
        anchors=anchors,
        enrich_address=False,
        request_delay_sec=0.0,
    )
    # Abort exactly at the threshold (3 consecutive failures).
    assert len(attempts) == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
    assert counters.errors_count == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
    assert counters.anchors_done == scrape_pipeline.YANDEX_SWEEP_MAX_CONSECUTIVE_FAILURES
    assert fake.banned is not None
    assert "consecutive" in fake.banned[0]
    assert fake.done is None  # aborted, never marked done


# ── Non-consecutive failures do not abort ───────────────────────────────────


async def test_non_consecutive_failures_do_not_abort(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A success between two failures resets the consecutive counter → no abort."""

    def _name_of(lat: float, anchors: list[tuple[float, float, str]]) -> str:
        for a_lat, _lon, nm in anchors:
            if a_lat == lat:
                return nm
        return "?"

    anchors = [
        *TEST_ANCHORS,
        (56.7700, 60.5500, "A4"),
        (56.8650, 60.6200, "A5"),
    ]
    failing = {"A1", "A3", "A5"}

    async def fake_fetch(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        if _name_of(lat, anchors) in failing:
            raise RuntimeError("transient")
        return [_fake_lot(f"{lat}-0")]

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=7,
        anchors=anchors,
        enrich_address=False,
        request_delay_sec=0.0,
    )
    assert counters.errors_count == 3  # A1, A3, A5
    assert counters.anchors_done == 5  # walked the whole list, no early abort
    assert counters.lots_fetched == 2  # A2 + A4 contributed
    assert fake.banned is None
    assert fake.done is not None


# ── mark_done вызывается даже при пустом SERP ───────────────────────────────


async def test_mark_done_always_called(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    async def fake_fetch_empty(
        self: YandexRealtyScraper,
        lat: float,
        lon: float,
        radius_m: int = 1000,
        max_pages: int = 2,
        **_: Any,
    ) -> list[ScrapedLot]:
        return []

    monkeypatch.setattr(YandexRealtyScraper, "fetch_around_multi_room", fake_fetch_empty)
    monkeypatch.setattr(scrape_pipeline, "save_listings", _fake_save)
    fake = _FakeRuns()
    _install_runs(monkeypatch, fake)

    counters = await scrape_pipeline.run_yandex_city_sweep(
        db=MagicMock(),
        run_id=8,
        anchors=TEST_ANCHORS,
        enrich_address=False,
        request_delay_sec=0.0,
    )

    assert fake.done is not None
    assert counters.lots_fetched == 0
    assert counters.anchors_done == len(TEST_ANCHORS)


# ── Admin endpoint tests (offline) ──────────────────────────────────────────


@pytest.fixture
def app_with_admin():
    from fastapi import FastAPI
    from fastapi.testclient import TestClient

    from app.api.v1 import admin as admin_module
    from app.core.db import get_db

    app = FastAPI()
    app.include_router(admin_module.router, prefix="/api/v1/admin")

    def fake_db():
        yield MagicMock()

    app.dependency_overrides[get_db] = fake_db
    return TestClient(app)


def test_yandex_start_endpoint_validates_pages_per_anchor(app_with_admin) -> None:
    """pages_per_anchor=99 превышает max=10 → 422."""
    r = app_with_admin.post(
        "/api/v1/admin/scrape/yandex-city-sweep",
        json={"pages_per_anchor": 99},
    )
    assert r.status_code == 422


def test_yandex_start_endpoint_validates_delay_too_low(app_with_admin) -> None:
    """request_delay_sec=1.0 меньше min=3.0 → 422."""
    r = app_with_admin.post(
        "/api/v1/admin/scrape/yandex-city-sweep",
        json={"request_delay_sec": 1.0},
    )
    assert r.status_code == 422


def test_yandex_start_endpoint_ok(app_with_admin) -> None:
    """Valid request → 200, run_id в ответе."""
    from unittest.mock import patch

    with (
        patch("app.services.scrape_runs.create_run", return_value=55),
        patch("app.services.scrape_pipeline.run_yandex_city_sweep", new_callable=AsyncMock),
    ):
        r = app_with_admin.post(
            "/api/v1/admin/scrape/yandex-city-sweep",
            json={"pages_per_anchor": 2, "enrich_address": True},
        )
    assert r.status_code == 200
    body = r.json()
    assert body["run_id"] == 55
    assert body["status"] == "running"
    assert body["pages_per_anchor"] == 2
    assert body["detail_top_n"] == 0


def test_yandex_cancel_endpoint(app_with_admin) -> None:
    """Cancel endpoint возвращает run_id + cancelled=True."""
    from unittest.mock import patch

    with patch("app.services.scrape_runs.mark_cancelled", return_value=True):
        r = app_with_admin.post("/api/v1/admin/scrape/yandex-city-sweep/99/cancel")
    assert r.status_code == 200
    body = r.json()
    assert body["run_id"] == 99
    assert body["cancelled"] is True
    assert body["ok"] is True


def test_yandex_cancel_endpoint_not_running(app_with_admin) -> None:
    """Cancel на не-running sweep → cancelled=False."""
    from unittest.mock import patch

    with patch("app.services.scrape_runs.mark_cancelled", return_value=False):
        r = app_with_admin.post("/api/v1/admin/scrape/yandex-city-sweep/100/cancel")
    assert r.status_code == 200
    assert r.json()["cancelled"] is False


def test_yandex_list_runs_endpoint(app_with_admin) -> None:
    """List runs endpoint возвращает список."""
    from datetime import UTC, datetime
    from unittest.mock import patch

    fake_rows = [
        {
            "run_id": 20,
            "source": "yandex_city_sweep",
            "status": "done",
            "params": {"pages_per_anchor": 2, "enrich_address": True},
            "counters": {"lots_fetched": 200, "address_enriched": 80},
            "error": None,
            "started_at": datetime(2025, 5, 1, tzinfo=UTC),
            "finished_at": datetime(2025, 5, 1, 1, tzinfo=UTC),
            "heartbeat_at": datetime(2025, 5, 1, 1, tzinfo=UTC),
        }
    ]
    with patch("app.services.scrape_runs.list_recent", return_value=fake_rows):
        r = app_with_admin.get("/api/v1/admin/scrape/yandex-city-sweep/runs")
    assert r.status_code == 200
    body = r.json()
    assert len(body) == 1
    assert body[0]["run_id"] == 20
    assert body[0]["source"] == "yandex_city_sweep"
    assert "2025-05-01" in body[0]["started_at"]