"""Offline tests для unified scrapers-страницы admin-API (epic консолидации UI). Покрытие 3 эндпоинтов (db/httpx мокаются, NO live network/DB): - GET /api/v1/admin/scrape/runs — unified runs + total + фильтры - GET /api/v1/admin/scraper/health — fetch_mode + browser + providers - POST /api/v1/admin/scraper/{src}/rotate-ip — changeip + no-rotate-url кейс """ from __future__ import annotations import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from datetime import UTC, datetime from typing import Any from unittest.mock import MagicMock, patch import pytest from fastapi import FastAPI from fastapi.testclient import TestClient @pytest.fixture def client() -> TestClient: from app.api.v1 import admin as admin_module from app.core.db import get_db app = FastAPI() app.include_router(admin_module.router, prefix="/api/v1/admin") def fake_db(): yield MagicMock() app.dependency_overrides[get_db] = fake_db return TestClient(app) # ── API 1: GET /scrape/runs ────────────────────────────────────────────────── def test_unified_runs_returns_rows_and_total(client: TestClient) -> None: """list_all → total + rows; ISO-форматирование дат + error_text/run_type.""" fake_rows = [ { "run_id": 5, "source": "avito_city_sweep", "run_type": "city_sweep", "status": "done", "params": {"pages_per_anchor": 3}, "counters": {"lots_fetched": 120}, "total_seen": 120, "new_count": 30, "started_at": datetime(2026, 6, 1, 10, tzinfo=UTC), "finished_at": datetime(2026, 6, 1, 11, tzinfo=UTC), "heartbeat_at": datetime(2026, 6, 1, 11, tzinfo=UTC), "error_text": None, } ] with patch("app.services.scrape_runs.list_all", return_value=(42, fake_rows)) as m: r = client.get("/api/v1/admin/scrape/runs") assert r.status_code == 200 body = r.json() assert body["total"] == 42 assert len(body["rows"]) == 1 row = body["rows"][0] assert row["run_id"] == 5 assert row["source"] == "avito_city_sweep" assert row["run_type"] == "city_sweep" assert row["total_seen"] == 120 assert row["new_count"] == 30 assert row["error_text"] is None assert "2026-06-01" in row["started_at"] # defaults передаются в сервис _, kwargs = m.call_args assert kwargs["source"] is None assert kwargs["status"] is None assert kwargs["limit"] == 50 assert kwargs["offset"] == 0 def test_unified_runs_passes_filters(client: TestClient) -> None: """source/status/limit/offset прокидываются в list_all.""" with patch("app.services.scrape_runs.list_all", return_value=(0, [])) as m: r = client.get( "/api/v1/admin/scrape/runs", params={ "source": "cian_city_sweep", "status": "banned", "limit": 10, "offset": 20, }, ) assert r.status_code == 200 _, kwargs = m.call_args assert kwargs["source"] == "cian_city_sweep" assert kwargs["status"] == "banned" assert kwargs["limit"] == 10 assert kwargs["offset"] == 20 def test_unified_runs_limit_over_max_422(client: TestClient) -> None: """limit=999 > max=200 → 422.""" r = client.get("/api/v1/admin/scrape/runs", params={"limit": 999}) assert r.status_code == 422 def test_unified_runs_negative_offset_422(client: TestClient) -> None: """offset=-1 < 0 → 422.""" r = client.get("/api/v1/admin/scrape/runs", params={"offset": -1}) assert r.status_code == 422 def test_unified_runs_invalid_status_422(client: TestClient) -> None: """status с непредусмотренным значением → 422 (Literal-валидация).""" r = client.get("/api/v1/admin/scrape/runs", params={"status": "unknown_status"}) assert r.status_code == 422 def test_unified_runs_valid_statuses_200(client: TestClient) -> None: """Все допустимые значения status проходят валидацию (ожидаем 200, не 422).""" valid_statuses = ["done", "running", "banned", "zombie", "failed", "cancelled"] for s in valid_statuses: with patch("app.services.scrape_runs.list_all", return_value=(0, [])): r = client.get("/api/v1/admin/scrape/runs", params={"status": s}) assert r.status_code == 200, f"status={s!r} должен возвращать 200, получили {r.status_code}" # ── API 2: GET /scraper/health ─────────────────────────────────────────────── def test_health_assembles_browser_and_providers(client: TestClient) -> None: """browser /health + ipify пробинг → fetch_mode + browser + 3 провайдера.""" from app.api.v1 import admin as admin_module async def fake_browser_health() -> admin_module.BrowserHealth: return admin_module.BrowserHealth( reachable=True, browsers={"avito": False, "cian": True, "yandex": False, "generic": True}, ) async def fake_current_ip(proxy_url: str | None) -> str | None: return "1.2.3.4" if proxy_url else None proxy_urls = { "avito": "http://u:p@host.a:10049", "cian": None, "yandex": "http://u:p@host.y:10051", } rotate_urls = {"avito": "http://ch/changeip", "cian": None, "yandex": None} with ( patch.object(admin_module, "_probe_browser_health", fake_browser_health), patch.object(admin_module, "_probe_current_ip", fake_current_ip), patch.object(admin_module, "_provider_proxy_url", lambda s: proxy_urls[s]), patch.object(admin_module, "_provider_rotate_url", lambda s: rotate_urls[s]), patch.object(admin_module.settings, "scraper_fetch_mode", "browser"), ): r = client.get("/api/v1/admin/scraper/health") assert r.status_code == 200 body = r.json() assert body["fetch_mode"] == "browser" assert body["browser"]["reachable"] is True assert body["browser"]["browsers"]["cian"] is True assert body["browser"]["browsers"]["avito"] is False by_source = {p["source"]: p for p in body["providers"]} assert set(by_source) == {"avito", "cian", "yandex"} assert by_source["avito"]["proxy_host"] == "host.a" assert by_source["avito"]["proxy_port"] == 10049 assert by_source["avito"]["rotate_supported"] is True assert by_source["avito"]["current_ip"] == "1.2.3.4" # cian без proxy → host/port/ip null, rotate_supported False assert by_source["cian"]["proxy_host"] is None assert by_source["cian"]["current_ip"] is None assert by_source["cian"]["rotate_supported"] is False # yandex имеет proxy но нет rotate_url assert by_source["yandex"]["proxy_host"] == "host.y" assert by_source["yandex"]["rotate_supported"] is False def test_health_browser_unreachable(client: TestClient) -> None: """httpx-ошибка на /health → reachable=False, browsers пуст.""" from app.api.v1 import admin as admin_module async def fake_current_ip(proxy_url: str | None) -> str | None: return None class _BoomClient: def __init__(self, *a: Any, **k: Any) -> None: pass async def __aenter__(self) -> _BoomClient: return self async def __aexit__(self, *a: Any) -> None: return None async def get(self, *a: Any, **k: Any) -> Any: raise RuntimeError("browser down") with ( patch.object(admin_module.httpx, "AsyncClient", _BoomClient), patch.object(admin_module, "_probe_current_ip", fake_current_ip), patch.object(admin_module, "_provider_proxy_url", lambda s: None), patch.object(admin_module, "_provider_rotate_url", lambda s: None), ): r = client.get("/api/v1/admin/scraper/health") assert r.status_code == 200 body = r.json() assert body["browser"]["reachable"] is False assert body["browser"]["browsers"] == {} # ── API 3: POST /scraper/{source}/rotate-ip ────────────────────────────────── def test_rotate_ip_calls_changeip(client: TestClient) -> None: """rotate_url задан → GET changeip + format=json, возврат new_ip из ответа.""" from app.api.v1 import admin as admin_module captured: dict[str, str] = {} class _FakeResp: def raise_for_status(self) -> None: return None def json(self) -> dict[str, str]: return {"new_ip": "9.9.9.9"} class _FakeClient: def __init__(self, *a: Any, **k: Any) -> None: pass async def __aenter__(self) -> _FakeClient: return self async def __aexit__(self, *a: Any) -> None: return None async def get(self, url: str, *a: Any, **k: Any) -> _FakeResp: captured["url"] = url return _FakeResp() with ( patch.object(admin_module.httpx, "AsyncClient", _FakeClient), patch.object(admin_module.settings, "avito_proxy_rotate_url", "http://ch/changeip?key=x"), ): r = client.post("/api/v1/admin/scraper/avito/rotate-ip") assert r.status_code == 200 body = r.json() assert body["ok"] is True assert body["new_ip"] == "9.9.9.9" assert "format=json" in captured["url"] # уже был '?' в URL → должен использовать '&' assert captured["url"].endswith("&format=json") def test_rotate_ip_no_rotate_url(client: TestClient) -> None: """rotate_url не задан → ok=False + reason про auto-rotate, changeip НЕ дёргается.""" from app.api.v1 import admin as admin_module with patch.object(admin_module.settings, "cian_proxy_rotate_url", None): r = client.post("/api/v1/admin/scraper/cian/rotate-ip") assert r.status_code == 200 body = r.json() assert body["ok"] is False assert body["new_ip"] is None assert "auto-rotate" in body["reason"] def test_rotate_ip_invalid_source_422(client: TestClient) -> None: """source вне {avito,cian,yandex} → 422 (Literal-валидация path).""" r = client.post("/api/v1/admin/scraper/domclick/rotate-ip") assert r.status_code == 422 def test_rotate_ip_changeip_error(client: TestClient) -> None: """httpx-ошибка changeip → ok=False + reason.""" from app.api.v1 import admin as admin_module class _BoomClient: def __init__(self, *a: Any, **k: Any) -> None: pass async def __aenter__(self) -> _BoomClient: return self async def __aexit__(self, *a: Any) -> None: return None async def get(self, *a: Any, **k: Any) -> Any: raise RuntimeError("changeip 500") with ( patch.object(admin_module.httpx, "AsyncClient", _BoomClient), patch.object(admin_module.settings, "yandex_proxy_rotate_url", "http://ch/changeip"), ): r = client.post("/api/v1/admin/scraper/yandex/rotate-ip") assert r.status_code == 200 body = r.json() assert body["ok"] is False assert "changeip error" in body["reason"]