gendesign/tradein-mvp/backend/tests/test_yandex_city_sweep.py

"""Offline unit tests for YandexCitySweepCounters + admin sweep endpoints (#861).

Pure & fast: NO live network, NO DB.

Legacy `run_yandex_city_sweep` orchestration behavior (SERP→save per-combo,
address-enrich phase, cooperative cancel, consecutive-SERP-failure abort, bounded
retry / combo-skip logging) удалён вместе с `app.services.scrape_pipeline` (#2397
Part E1) — эквивалентная regression-coverage теперь в
`test_scraper_kit_pipeline_parity2.py::test_yandex_city_sweep` (kit
`run_yandex_city_sweep`).

Coverage:
  - YandexCitySweepCounters корректно заполняются (dataclass shape)
  - Combos watchdog timeout существенно больше ANCHOR_TIMEOUT_SEC (arithmetic-only)
  - Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI
    (config/matcher/enrichment)
"""

from __future__ import annotations

import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from datetime import UTC
from unittest.mock import AsyncMock

import pytest

# ── YandexCitySweepCounters ─────────────────────────────────────────────────


def test_counters_defaults() -> None:
    from scraper_kit.orchestration.pipeline import YandexCitySweepCounters

    c = YandexCitySweepCounters()
    assert c.anchors_total == 0
    assert c.lots_fetched == 0
    assert c.combos_skipped == 0
    assert c.address_attempted == 0
    assert c.address_enriched == 0
    assert c.address_failed == 0
    assert c.errors_count == 0


def test_counters_to_dict_all_keys() -> None:
    from dataclasses import fields

    from scraper_kit.orchestration.pipeline import YandexCitySweepCounters

    c = YandexCitySweepCounters()
    d = c.to_dict()
    expected = {f.name for f in fields(c)}
    assert set(d.keys()) == expected


# ── Admin endpoint tests (offline) ──────────────────────────────────────────


@pytest.fixture
def app_with_admin():
    from unittest.mock import MagicMock

    from fastapi import FastAPI
    from fastapi.testclient import TestClient

    from app.api.v1 import admin as admin_module
    from app.core.db import get_db

    app = FastAPI()
    app.include_router(admin_module.router, prefix="/api/v1/admin")

    def fake_db():
        yield MagicMock()

    app.dependency_overrides[get_db] = fake_db
    return TestClient(app)


def test_yandex_start_endpoint_validates_pages_per_anchor(app_with_admin) -> None:
    """pages_per_anchor=99 превышает max=10 → 422."""
    r = app_with_admin.post(
        "/api/v1/admin/scrape/yandex-city-sweep",
        json={"pages_per_anchor": 99},
    )
    assert r.status_code == 422


def test_yandex_start_endpoint_validates_delay_too_low(app_with_admin) -> None:
    """request_delay_sec=1.0 меньше min=3.0 → 422."""
    r = app_with_admin.post(
        "/api/v1/admin/scrape/yandex-city-sweep",
        json={"request_delay_sec": 1.0},
    )
    assert r.status_code == 422


def test_yandex_start_endpoint_ok(app_with_admin) -> None:
    """Valid request → 200, run_id в ответе; kit sweep вызван с DI (config/matcher/enrichment)."""
    from unittest.mock import patch

    from app.services.scraper_adapters import (
        RealEnrichmentJobs,
        RealMatcherAdapter,
        RealScraperConfig,
    )

    with (
        patch("app.services.scrape_runs.create_run", return_value=55),
        patch("app.api.v1.admin.run_yandex_city_sweep", new_callable=AsyncMock) as mock_sweep,
    ):
        r = app_with_admin.post(
            "/api/v1/admin/scrape/yandex-city-sweep",
            json={"pages_per_anchor": 2, "enrich_address": True},
        )
    assert r.status_code == 200
    body = r.json()
    assert body["run_id"] == 55
    assert body["status"] == "running"
    assert body["pages_per_anchor"] == 2
    assert body["detail_top_n"] == 0

    mock_sweep.assert_called_once()
    kw = mock_sweep.call_args.kwargs
    assert isinstance(kw["config"], RealScraperConfig)
    assert isinstance(kw["matcher"], RealMatcherAdapter)
    assert isinstance(kw["enrichment"], RealEnrichmentJobs)
    assert "proxy_provider" in kw


def test_yandex_cancel_endpoint(app_with_admin) -> None:
    """Cancel endpoint возвращает run_id + cancelled=True."""
    from unittest.mock import patch

    with patch("app.services.scrape_runs.mark_cancelled", return_value=True):
        r = app_with_admin.post("/api/v1/admin/scrape/yandex-city-sweep/99/cancel")
    assert r.status_code == 200
    body = r.json()
    assert body["run_id"] == 99
    assert body["cancelled"] is True
    assert body["ok"] is True


def test_yandex_cancel_endpoint_not_running(app_with_admin) -> None:
    """Cancel на не-running sweep → cancelled=False."""
    from unittest.mock import patch

    with patch("app.services.scrape_runs.mark_cancelled", return_value=False):
        r = app_with_admin.post("/api/v1/admin/scrape/yandex-city-sweep/100/cancel")
    assert r.status_code == 200
    assert r.json()["cancelled"] is False


def test_yandex_list_runs_endpoint(app_with_admin) -> None:
    """List runs endpoint возвращает список."""
    from datetime import datetime
    from unittest.mock import patch

    fake_rows = [
        {
            "run_id": 20,
            "source": "yandex_city_sweep",
            "status": "done",
            "params": {"pages_per_anchor": 2, "enrich_address": True},
            "counters": {"lots_fetched": 200, "address_enriched": 80},
            "error": None,
            "started_at": datetime(2025, 5, 1, tzinfo=UTC),
            "finished_at": datetime(2025, 5, 1, 1, tzinfo=UTC),
            "heartbeat_at": datetime(2025, 5, 1, 1, tzinfo=UTC),
        }
    ]
    with patch("app.services.scrape_runs.list_recent", return_value=fake_rows):
        r = app_with_admin.get("/api/v1/admin/scrape/yandex-city-sweep/runs")
    assert r.status_code == 200
    body = r.json()
    assert len(body) == 1
    assert body[0]["run_id"] == 20
    assert body[0]["source"] == "yandex_city_sweep"
    assert "2025-05-01" in body[0]["started_at"]


# ── Watchdog timeout: combos-mode budget dwarfs ANCHOR_TIMEOUT_SEC ──────────


def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None:
    """В combos/center mode (anchors=None) _sweep_timeout намного > ANCHOR_TIMEOUT_SEC.

    Проверяет что вычисленный таймаут для дефолтных 30 combos × 3 pages > 1500s,
    т.е. не равен ANCHOR_TIMEOUT_SEC=240 и обеспечивает полный прогон sweep'а.
    Не нужны мокапы — только публичные константы и арифметика из kit-модуля.
    """
    from scraper_kit.orchestration.pipeline import (
        _YANDEX_ADDRESS_ENRICH_BUDGET_S,
        _YANDEX_COMBOS_PER_FETCH_S,
        ANCHOR_TIMEOUT_SEC,
    )

    from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH

    # Параметры prod-режима
    rooms_list = list(ROOM_PATH.keys())  # 5 комнатностей
    price_ranges = DEFAULT_PRICE_RANGES  # 6 ценовых диапазонов
    max_pages = 3
    request_delay_sec = 9.0  # продакшн default

    num_combos = len(rooms_list) * len(price_ranges)
    assert num_combos == 30, f"Expected 30 combos, got {num_combos}"

    sweep_timeout = max(
        ANCHOR_TIMEOUT_SEC,
        num_combos * max_pages * (request_delay_sec + _YANDEX_COMBOS_PER_FETCH_S)
        + _YANDEX_ADDRESS_ENRICH_BUDGET_S,
    )

    # Должен быть существенно > 1500s (≈ 25 мин), что покрывает полный run.
    assert sweep_timeout > 1500, (
        f"sweep_timeout={sweep_timeout:.0f}s unexpectedly small — "
        f"combos-mode will still timeout mid-sweep"
    )
    # И намного > ANCHOR_TIMEOUT_SEC (240s)
    assert (
        sweep_timeout > ANCHOR_TIMEOUT_SEC * 4
    ), f"sweep_timeout={sweep_timeout:.0f}s should be >> ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC}s"