"""Offline unit tests for YandexCitySweepCounters + admin sweep endpoints (#861). Pure & fast: NO live network, NO DB. Legacy `run_yandex_city_sweep` orchestration behavior (SERP→save per-combo, address-enrich phase, cooperative cancel, consecutive-SERP-failure abort, bounded retry / combo-skip logging) удалён вместе с `app.services.scrape_pipeline` (#2397 Part E1) — эквивалентная regression-coverage теперь в `test_scraper_kit_pipeline_parity2.py::test_yandex_city_sweep` (kit `run_yandex_city_sweep`). Coverage: - YandexCitySweepCounters корректно заполняются (dataclass shape) - Combos watchdog timeout существенно больше ANCHOR_TIMEOUT_SEC (arithmetic-only; `DEFAULT_PRICE_RANGES`/`ROOM_PATH` — kit `scraper_kit.providers.yandex.serp`, ретаргетировано с удалённого `app.services.scrapers.yandex_realty`, #2397 Part E2) - Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI (config/matcher/enrichment) """ from __future__ import annotations import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from datetime import UTC from unittest.mock import AsyncMock import pytest # ── YandexCitySweepCounters ───────────────────────────────────────────────── def test_counters_defaults() -> None: from scraper_kit.orchestration.pipeline import YandexCitySweepCounters c = YandexCitySweepCounters() assert c.anchors_total == 0 assert c.lots_fetched == 0 assert c.combos_skipped == 0 assert c.address_attempted == 0 assert c.address_enriched == 0 assert c.address_failed == 0 assert c.errors_count == 0 def test_counters_to_dict_all_keys() -> None: from dataclasses import fields from scraper_kit.orchestration.pipeline import YandexCitySweepCounters c = YandexCitySweepCounters() d = c.to_dict() expected = {f.name for f in fields(c)} assert set(d.keys()) == expected # ── Admin endpoint tests (offline) ────────────────────────────────────────── @pytest.fixture def app_with_admin(): from unittest.mock import MagicMock from fastapi import FastAPI from fastapi.testclient import TestClient from app.api.v1 import admin as admin_module from app.core.db import get_db app = FastAPI() app.include_router(admin_module.router, prefix="/api/v1/admin") def fake_db(): yield MagicMock() app.dependency_overrides[get_db] = fake_db return TestClient(app) def test_yandex_start_endpoint_validates_pages_per_anchor(app_with_admin) -> None: """pages_per_anchor=99 превышает max=10 → 422.""" r = app_with_admin.post( "/api/v1/admin/scrape/yandex-city-sweep", json={"pages_per_anchor": 99}, ) assert r.status_code == 422 def test_yandex_start_endpoint_validates_delay_too_low(app_with_admin) -> None: """request_delay_sec=1.0 меньше min=3.0 → 422.""" r = app_with_admin.post( "/api/v1/admin/scrape/yandex-city-sweep", json={"request_delay_sec": 1.0}, ) assert r.status_code == 422 def test_yandex_start_endpoint_ok(app_with_admin) -> None: """Valid request → 200, run_id в ответе; kit sweep вызван с DI (config/matcher/enrichment).""" from unittest.mock import patch from app.services.scraper_adapters import ( RealEnrichmentJobs, RealMatcherAdapter, RealScraperConfig, ) with ( patch("app.services.scrape_runs.create_run", return_value=55), patch("app.api.v1.admin.run_yandex_city_sweep", new_callable=AsyncMock) as mock_sweep, ): r = app_with_admin.post( "/api/v1/admin/scrape/yandex-city-sweep", json={"pages_per_anchor": 2, "enrich_address": True}, ) assert r.status_code == 200 body = r.json() assert body["run_id"] == 55 assert body["status"] == "running" assert body["pages_per_anchor"] == 2 assert body["detail_top_n"] == 0 mock_sweep.assert_called_once() kw = mock_sweep.call_args.kwargs assert isinstance(kw["config"], RealScraperConfig) assert isinstance(kw["matcher"], RealMatcherAdapter) assert isinstance(kw["enrichment"], RealEnrichmentJobs) assert "proxy_provider" in kw def test_yandex_cancel_endpoint(app_with_admin) -> None: """Cancel endpoint возвращает run_id + cancelled=True.""" from unittest.mock import patch with patch("app.services.scrape_runs.mark_cancelled", return_value=True): r = app_with_admin.post("/api/v1/admin/scrape/yandex-city-sweep/99/cancel") assert r.status_code == 200 body = r.json() assert body["run_id"] == 99 assert body["cancelled"] is True assert body["ok"] is True def test_yandex_cancel_endpoint_not_running(app_with_admin) -> None: """Cancel на не-running sweep → cancelled=False.""" from unittest.mock import patch with patch("app.services.scrape_runs.mark_cancelled", return_value=False): r = app_with_admin.post("/api/v1/admin/scrape/yandex-city-sweep/100/cancel") assert r.status_code == 200 assert r.json()["cancelled"] is False def test_yandex_list_runs_endpoint(app_with_admin) -> None: """List runs endpoint возвращает список.""" from datetime import datetime from unittest.mock import patch fake_rows = [ { "run_id": 20, "source": "yandex_city_sweep", "status": "done", "params": {"pages_per_anchor": 2, "enrich_address": True}, "counters": {"lots_fetched": 200, "address_enriched": 80}, "error": None, "started_at": datetime(2025, 5, 1, tzinfo=UTC), "finished_at": datetime(2025, 5, 1, 1, tzinfo=UTC), "heartbeat_at": datetime(2025, 5, 1, 1, tzinfo=UTC), } ] with patch("app.services.scrape_runs.list_recent", return_value=fake_rows): r = app_with_admin.get("/api/v1/admin/scrape/yandex-city-sweep/runs") assert r.status_code == 200 body = r.json() assert len(body) == 1 assert body[0]["run_id"] == 20 assert body[0]["source"] == "yandex_city_sweep" assert "2025-05-01" in body[0]["started_at"] # ── Watchdog timeout: combos-mode budget dwarfs ANCHOR_TIMEOUT_SEC ────────── def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None: """В combos/center mode (anchors=None) _sweep_timeout намного > ANCHOR_TIMEOUT_SEC. Проверяет что вычисленный таймаут для дефолтных 30 combos × 3 pages > 1500s, т.е. не равен ANCHOR_TIMEOUT_SEC=240 и обеспечивает полный прогон sweep'а. Не нужны мокапы — только публичные константы и арифметика из kit-модуля. """ from scraper_kit.orchestration.pipeline import ( _YANDEX_ADDRESS_ENRICH_BUDGET_S, _YANDEX_COMBOS_PER_FETCH_S, ANCHOR_TIMEOUT_SEC, ) from scraper_kit.providers.yandex.serp import DEFAULT_PRICE_RANGES, ROOM_PATH # Параметры prod-режима rooms_list = list(ROOM_PATH.keys()) # 5 комнатностей price_ranges = DEFAULT_PRICE_RANGES # 6 ценовых диапазонов max_pages = 3 request_delay_sec = 9.0 # продакшн default num_combos = len(rooms_list) * len(price_ranges) assert num_combos == 30, f"Expected 30 combos, got {num_combos}" sweep_timeout = max( ANCHOR_TIMEOUT_SEC, num_combos * max_pages * (request_delay_sec + _YANDEX_COMBOS_PER_FETCH_S) + _YANDEX_ADDRESS_ENRICH_BUDGET_S, ) # Должен быть существенно > 1500s (≈ 25 мин), что покрывает полный run. assert sweep_timeout > 1500, ( f"sweep_timeout={sweep_timeout:.0f}s unexpectedly small — " f"combos-mode will still timeout mid-sweep" ) # И намного > ANCHOR_TIMEOUT_SEC (240s) assert ( sweep_timeout > ANCHOR_TIMEOUT_SEC * 4 ), f"sweep_timeout={sweep_timeout:.0f}s should be >> ANCHOR_TIMEOUT_SEC={ANCHOR_TIMEOUT_SEC}s"