"""Offline unit tests for CianCitySweepCounters + admin sweep endpoints (#860). Pure & fast: NO live network, NO DB. Legacy `run_cian_city_sweep` orchestration behavior (SERP→save→detail→houses phase sequencing, cooperative cancel, consecutive-SERP-failure abort, per-item error tolerance) удалён вместе с `app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная regression-coverage теперь в `test_scraper_kit_pipeline_parity2.py::test_cian_city_sweep` (kit `run_cian_city_sweep`). Coverage: - CianCitySweepCounters корректно заполняются (dataclass shape) - _cian_anchor_timeout_s watchdog-формула (#2160) - Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI (config/matcher) """ from __future__ import annotations import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") import pytest # ── CianCitySweepCounters ─────────────────────────────────────────────────── def test_cian_sweep_counters_defaults() -> None: from scraper_kit.orchestration.pipeline import CianCitySweepCounters c = CianCitySweepCounters() assert c.anchors_total == 0 assert c.lots_fetched == 0 assert c.detail_attempted == 0 assert c.houses_attempted == 0 assert c.errors_count == 0 def test_cian_sweep_counters_to_dict_all_keys() -> None: from dataclasses import fields from scraper_kit.orchestration.pipeline import CianCitySweepCounters c = CianCitySweepCounters() d = c.to_dict() expected = {f.name for f in fields(c)} assert set(d.keys()) == expected # ── Admin endpoint tests (offline) ────────────────────────────────────────── @pytest.fixture def app_with_admin(): from unittest.mock import MagicMock from fastapi import FastAPI from fastapi.testclient import TestClient from app.api.v1 import admin as admin_module from app.core.db import get_db app = FastAPI() app.include_router(admin_module.router, prefix="/api/v1/admin") def fake_db(): yield MagicMock() app.dependency_overrides[get_db] = fake_db return TestClient(app) def test_cian_start_endpoint_validates_pages_per_anchor(app_with_admin) -> None: """pages_per_anchor=99 превышает max=10 → 422.""" r = app_with_admin.post( "/api/v1/admin/scrape/cian-city-sweep", json={"pages_per_anchor": 99}, ) assert r.status_code == 422 def test_cian_start_endpoint_validates_delay_too_low(app_with_admin) -> None: """request_delay_sec=1.0 меньше min=3.0 → 422.""" r = app_with_admin.post( "/api/v1/admin/scrape/cian-city-sweep", json={"request_delay_sec": 1.0}, ) assert r.status_code == 422 def test_cian_start_endpoint_ok(app_with_admin) -> None: """Valid request → 200, run_id; kit sweep вызван с DI (config/matcher, БЕЗ enrichment).""" from unittest.mock import AsyncMock, patch from app.services.scraper_adapters import RealMatcherAdapter, RealScraperConfig with ( patch("app.services.scrape_runs.create_run", return_value=42), patch("app.api.v1.admin.run_cian_city_sweep", new_callable=AsyncMock) as mock_sweep, ): r = app_with_admin.post( "/api/v1/admin/scrape/cian-city-sweep", json={"pages_per_anchor": 2, "detail_top_n": 5, "enrich_houses": False}, ) assert r.status_code == 200 body = r.json() assert body["run_id"] == 42 assert body["status"] == "running" assert body["pages_per_anchor"] == 2 assert body["detail_top_n"] == 5 mock_sweep.assert_called_once() kw = mock_sweep.call_args.kwargs assert isinstance(kw["config"], RealScraperConfig) assert isinstance(kw["matcher"], RealMatcherAdapter) assert "proxy_provider" in kw assert "enrichment" not in kw def test_cian_cancel_endpoint(app_with_admin) -> None: """Cancel endpoint возвращает run_id + cancelled=True.""" from unittest.mock import patch with patch("app.services.scrape_runs.mark_cancelled", return_value=True): r = app_with_admin.post("/api/v1/admin/scrape/cian-city-sweep/77/cancel") assert r.status_code == 200 body = r.json() assert body["run_id"] == 77 assert body["cancelled"] is True assert body["ok"] is True def test_cian_cancel_endpoint_not_running(app_with_admin) -> None: """Cancel на не-running sweep → cancelled=False.""" from unittest.mock import patch with patch("app.services.scrape_runs.mark_cancelled", return_value=False): r = app_with_admin.post("/api/v1/admin/scrape/cian-city-sweep/88/cancel") assert r.status_code == 200 assert r.json()["cancelled"] is False def test_cian_list_runs_endpoint(app_with_admin) -> None: """List runs endpoint возвращает список.""" from datetime import UTC, datetime from unittest.mock import patch fake_rows = [ { "run_id": 10, "source": "cian_city_sweep", "status": "done", "params": {"pages_per_anchor": 3}, "counters": {"lots_fetched": 150}, "error": None, "started_at": datetime(2025, 5, 1, tzinfo=UTC), "finished_at": datetime(2025, 5, 1, 1, tzinfo=UTC), "heartbeat_at": datetime(2025, 5, 1, 1, tzinfo=UTC), } ] with patch("app.services.scrape_runs.list_recent", return_value=fake_rows): r = app_with_admin.get("/api/v1/admin/scrape/cian-city-sweep/runs") assert r.status_code == 200 body = r.json() assert len(body) == 1 assert body[0]["run_id"] == 10 assert body[0]["source"] == "cian_city_sweep" assert "2025-05-01" in body[0]["started_at"] # ── #2160: масштабируемый per-anchor watchdog для proxy-pool browser ────────── def test_cian_anchor_timeout_defaults() -> None: """Формула watchdog даёт worst-case ~1580s при дефолтных параметрах. 4 room-buckets × 3 pages × (5s delay + 70s serp-fetch) + 10 detail × 50s + 180s houses = 900 + 500 + 180 = 1580s. Фиксированный ANCHOR_TIMEOUT_SEC=240 гильотинил бы якорь в SERP-фазе (#2160): каждый SERP-фетч через camoufox 13-45s. """ from scraper_kit.orchestration.pipeline import ANCHOR_TIMEOUT_SEC, _cian_anchor_timeout_s t = _cian_anchor_timeout_s( pages_per_anchor=3, request_delay_sec=5.0, detail_top_n=10, enrich_houses=True, ) assert t == pytest.approx(1580.0) assert t > ANCHOR_TIMEOUT_SEC def test_cian_anchor_timeout_no_houses() -> None: """enrich_houses=False убирает houses-бюджет (180s) из формулы.""" from scraper_kit.orchestration.pipeline import _cian_anchor_timeout_s t = _cian_anchor_timeout_s( pages_per_anchor=3, request_delay_sec=5.0, detail_top_n=10, enrich_houses=False, ) assert t == pytest.approx(1400.0) # 900 + 500 + 0 def test_cian_anchor_timeout_floor_at_anchor_timeout_sec() -> None: """Малые параметры не опускают таймаут ниже ANCHOR_TIMEOUT_SEC (max-ветка).""" from scraper_kit.orchestration.pipeline import ANCHOR_TIMEOUT_SEC, _cian_anchor_timeout_s t = _cian_anchor_timeout_s( pages_per_anchor=0, request_delay_sec=0.0, detail_top_n=0, enrich_houses=False, ) assert t == float(ANCHOR_TIMEOUT_SEC)