gendesign/tradein-mvp/backend/tests/test_cian_city_sweep.py

"""Offline unit tests for CianCitySweepCounters + admin sweep endpoints (#860).

Pure & fast: NO live network, NO DB.

Legacy `run_cian_city_sweep` orchestration behavior (SERP→save→detail→houses phase
sequencing, cooperative cancel, consecutive-SERP-failure abort, per-item error
tolerance) удалён вместе с `app.services.scrape_pipeline` (#2397 Part E1) —
эквивалентная regression-coverage теперь в
`test_scraper_kit_pipeline_parity2.py::test_cian_city_sweep` (kit
`run_cian_city_sweep`).

Coverage:
  - CianCitySweepCounters корректно заполняются (dataclass shape)
  - _cian_anchor_timeout_s watchdog-формула (#2160)
  - Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI (config/matcher)
"""

from __future__ import annotations

import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

import pytest

# ── CianCitySweepCounters ───────────────────────────────────────────────────


def test_cian_sweep_counters_defaults() -> None:
    from scraper_kit.orchestration.pipeline import CianCitySweepCounters

    c = CianCitySweepCounters()
    assert c.anchors_total == 0
    assert c.lots_fetched == 0
    assert c.detail_attempted == 0
    assert c.houses_attempted == 0
    assert c.errors_count == 0


def test_cian_sweep_counters_to_dict_all_keys() -> None:
    from dataclasses import fields

    from scraper_kit.orchestration.pipeline import CianCitySweepCounters

    c = CianCitySweepCounters()
    d = c.to_dict()
    expected = {f.name for f in fields(c)}
    assert set(d.keys()) == expected


# ── Admin endpoint tests (offline) ──────────────────────────────────────────


@pytest.fixture
def app_with_admin():
    from unittest.mock import MagicMock

    from fastapi import FastAPI
    from fastapi.testclient import TestClient

    from app.api.v1 import admin as admin_module
    from app.core.db import get_db

    app = FastAPI()
    app.include_router(admin_module.router, prefix="/api/v1/admin")

    def fake_db():
        yield MagicMock()

    app.dependency_overrides[get_db] = fake_db
    return TestClient(app)


def test_cian_start_endpoint_validates_pages_per_anchor(app_with_admin) -> None:
    """pages_per_anchor=99 превышает max=10 → 422."""
    r = app_with_admin.post(
        "/api/v1/admin/scrape/cian-city-sweep",
        json={"pages_per_anchor": 99},
    )
    assert r.status_code == 422


def test_cian_start_endpoint_validates_delay_too_low(app_with_admin) -> None:
    """request_delay_sec=1.0 меньше min=3.0 → 422."""
    r = app_with_admin.post(
        "/api/v1/admin/scrape/cian-city-sweep",
        json={"request_delay_sec": 1.0},
    )
    assert r.status_code == 422


def test_cian_start_endpoint_ok(app_with_admin) -> None:
    """Valid request → 200, run_id; kit sweep вызван с DI (config/matcher, БЕЗ enrichment)."""
    from unittest.mock import AsyncMock, patch

    from app.services.scraper_adapters import RealMatcherAdapter, RealScraperConfig

    with (
        patch("app.services.scrape_runs.create_run", return_value=42),
        patch("app.api.v1.admin.run_cian_city_sweep", new_callable=AsyncMock) as mock_sweep,
    ):
        r = app_with_admin.post(
            "/api/v1/admin/scrape/cian-city-sweep",
            json={"pages_per_anchor": 2, "detail_top_n": 5, "enrich_houses": False},
        )
    assert r.status_code == 200
    body = r.json()
    assert body["run_id"] == 42
    assert body["status"] == "running"
    assert body["pages_per_anchor"] == 2
    assert body["detail_top_n"] == 5

    mock_sweep.assert_called_once()
    kw = mock_sweep.call_args.kwargs
    assert isinstance(kw["config"], RealScraperConfig)
    assert isinstance(kw["matcher"], RealMatcherAdapter)
    assert "proxy_provider" in kw
    assert "enrichment" not in kw


def test_cian_cancel_endpoint(app_with_admin) -> None:
    """Cancel endpoint возвращает run_id + cancelled=True."""
    from unittest.mock import patch

    with patch("app.services.scrape_runs.mark_cancelled", return_value=True):
        r = app_with_admin.post("/api/v1/admin/scrape/cian-city-sweep/77/cancel")
    assert r.status_code == 200
    body = r.json()
    assert body["run_id"] == 77
    assert body["cancelled"] is True
    assert body["ok"] is True


def test_cian_cancel_endpoint_not_running(app_with_admin) -> None:
    """Cancel на не-running sweep → cancelled=False."""
    from unittest.mock import patch

    with patch("app.services.scrape_runs.mark_cancelled", return_value=False):
        r = app_with_admin.post("/api/v1/admin/scrape/cian-city-sweep/88/cancel")
    assert r.status_code == 200
    assert r.json()["cancelled"] is False


def test_cian_list_runs_endpoint(app_with_admin) -> None:
    """List runs endpoint возвращает список."""
    from datetime import UTC, datetime
    from unittest.mock import patch

    fake_rows = [
        {
            "run_id": 10,
            "source": "cian_city_sweep",
            "status": "done",
            "params": {"pages_per_anchor": 3},
            "counters": {"lots_fetched": 150},
            "error": None,
            "started_at": datetime(2025, 5, 1, tzinfo=UTC),
            "finished_at": datetime(2025, 5, 1, 1, tzinfo=UTC),
            "heartbeat_at": datetime(2025, 5, 1, 1, tzinfo=UTC),
        }
    ]
    with patch("app.services.scrape_runs.list_recent", return_value=fake_rows):
        r = app_with_admin.get("/api/v1/admin/scrape/cian-city-sweep/runs")
    assert r.status_code == 200
    body = r.json()
    assert len(body) == 1
    assert body[0]["run_id"] == 10
    assert body[0]["source"] == "cian_city_sweep"
    assert "2025-05-01" in body[0]["started_at"]


# ── #2160: масштабируемый per-anchor watchdog для proxy-pool browser ──────────


def test_cian_anchor_timeout_defaults() -> None:
    """Формула watchdog даёт worst-case ~1580s при дефолтных параметрах.

    4 room-buckets × 3 pages × (5s delay + 70s serp-fetch) + 10 detail × 50s + 180s houses
    = 900 + 500 + 180 = 1580s. Фиксированный ANCHOR_TIMEOUT_SEC=240 гильотинил бы якорь в
    SERP-фазе (#2160): каждый SERP-фетч через camoufox 13-45s.
    """
    from scraper_kit.orchestration.pipeline import ANCHOR_TIMEOUT_SEC, _cian_anchor_timeout_s

    t = _cian_anchor_timeout_s(
        pages_per_anchor=3,
        request_delay_sec=5.0,
        detail_top_n=10,
        enrich_houses=True,
    )
    assert t == pytest.approx(1580.0)
    assert t > ANCHOR_TIMEOUT_SEC


def test_cian_anchor_timeout_no_houses() -> None:
    """enrich_houses=False убирает houses-бюджет (180s) из формулы."""
    from scraper_kit.orchestration.pipeline import _cian_anchor_timeout_s

    t = _cian_anchor_timeout_s(
        pages_per_anchor=3,
        request_delay_sec=5.0,
        detail_top_n=10,
        enrich_houses=False,
    )
    assert t == pytest.approx(1400.0)  # 900 + 500 + 0


def test_cian_anchor_timeout_floor_at_anchor_timeout_sec() -> None:
    """Малые параметры не опускают таймаут ниже ANCHOR_TIMEOUT_SEC (max-ветка)."""
    from scraper_kit.orchestration.pipeline import ANCHOR_TIMEOUT_SEC, _cian_anchor_timeout_s

    t = _cian_anchor_timeout_s(
        pages_per_anchor=0,
        request_delay_sec=0.0,
        detail_top_n=0,
        enrich_houses=False,
    )
    assert t == float(ANCHOR_TIMEOUT_SEC)