bot-backend
8361961a44
- pyproject.toml: httpx[socks]>=0.27.0 — тянет socksio, socks5:// прокси в _probe_current_ip работает прозрачно - admin.py list_scrape_runs_unified: status Literal["done","running","banned","zombie","failed","cancelled"] вместо str|None — невалидные значения → 422 - avito_houses.py save_house_catalog_enrichment: гард ext_id=0 + нет адреса → skip without DB touch; два id-less novostroyka URL не схлопываются в bogus avito:0 в house_sources - тесты: test_unified_runs_invalid_status_422 + test_unified_runs_valid_statuses_200; test_save_house_catalog_enrichment_skips_zero_ext_id_no_address + *_with_address_persists
308 lines
11 KiB
Python
308 lines
11 KiB
Python
"""Offline tests для unified scrapers-страницы admin-API (epic консолидации UI).
|
||
|
||
Покрытие 3 эндпоинтов (db/httpx мокаются, NO live network/DB):
|
||
- GET /api/v1/admin/scrape/runs — unified runs + total + фильтры
|
||
- GET /api/v1/admin/scraper/health — fetch_mode + browser + providers
|
||
- POST /api/v1/admin/scraper/{src}/rotate-ip — changeip + no-rotate-url кейс
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from datetime import UTC, datetime
|
||
from typing import Any
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
import pytest
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
|
||
|
||
@pytest.fixture
|
||
def client() -> TestClient:
|
||
from app.api.v1 import admin as admin_module
|
||
from app.core.db import get_db
|
||
|
||
app = FastAPI()
|
||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||
|
||
def fake_db():
|
||
yield MagicMock()
|
||
|
||
app.dependency_overrides[get_db] = fake_db
|
||
return TestClient(app)
|
||
|
||
|
||
# ── API 1: GET /scrape/runs ──────────────────────────────────────────────────
|
||
|
||
|
||
def test_unified_runs_returns_rows_and_total(client: TestClient) -> None:
|
||
"""list_all → total + rows; ISO-форматирование дат + error_text/run_type."""
|
||
fake_rows = [
|
||
{
|
||
"run_id": 5,
|
||
"source": "avito_city_sweep",
|
||
"run_type": "city_sweep",
|
||
"status": "done",
|
||
"params": {"pages_per_anchor": 3},
|
||
"counters": {"lots_fetched": 120},
|
||
"total_seen": 120,
|
||
"new_count": 30,
|
||
"started_at": datetime(2026, 6, 1, 10, tzinfo=UTC),
|
||
"finished_at": datetime(2026, 6, 1, 11, tzinfo=UTC),
|
||
"heartbeat_at": datetime(2026, 6, 1, 11, tzinfo=UTC),
|
||
"error_text": None,
|
||
}
|
||
]
|
||
with patch("app.services.scrape_runs.list_all", return_value=(42, fake_rows)) as m:
|
||
r = client.get("/api/v1/admin/scrape/runs")
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["total"] == 42
|
||
assert len(body["rows"]) == 1
|
||
row = body["rows"][0]
|
||
assert row["run_id"] == 5
|
||
assert row["source"] == "avito_city_sweep"
|
||
assert row["run_type"] == "city_sweep"
|
||
assert row["total_seen"] == 120
|
||
assert row["new_count"] == 30
|
||
assert row["error_text"] is None
|
||
assert "2026-06-01" in row["started_at"]
|
||
# defaults передаются в сервис
|
||
_, kwargs = m.call_args
|
||
assert kwargs["source"] is None
|
||
assert kwargs["status"] is None
|
||
assert kwargs["limit"] == 50
|
||
assert kwargs["offset"] == 0
|
||
|
||
|
||
def test_unified_runs_passes_filters(client: TestClient) -> None:
|
||
"""source/status/limit/offset прокидываются в list_all."""
|
||
with patch("app.services.scrape_runs.list_all", return_value=(0, [])) as m:
|
||
r = client.get(
|
||
"/api/v1/admin/scrape/runs",
|
||
params={
|
||
"source": "cian_city_sweep",
|
||
"status": "banned",
|
||
"limit": 10,
|
||
"offset": 20,
|
||
},
|
||
)
|
||
assert r.status_code == 200
|
||
_, kwargs = m.call_args
|
||
assert kwargs["source"] == "cian_city_sweep"
|
||
assert kwargs["status"] == "banned"
|
||
assert kwargs["limit"] == 10
|
||
assert kwargs["offset"] == 20
|
||
|
||
|
||
def test_unified_runs_limit_over_max_422(client: TestClient) -> None:
|
||
"""limit=999 > max=200 → 422."""
|
||
r = client.get("/api/v1/admin/scrape/runs", params={"limit": 999})
|
||
assert r.status_code == 422
|
||
|
||
|
||
def test_unified_runs_negative_offset_422(client: TestClient) -> None:
|
||
"""offset=-1 < 0 → 422."""
|
||
r = client.get("/api/v1/admin/scrape/runs", params={"offset": -1})
|
||
assert r.status_code == 422
|
||
|
||
|
||
def test_unified_runs_invalid_status_422(client: TestClient) -> None:
|
||
"""status с непредусмотренным значением → 422 (Literal-валидация)."""
|
||
r = client.get("/api/v1/admin/scrape/runs", params={"status": "unknown_status"})
|
||
assert r.status_code == 422
|
||
|
||
|
||
def test_unified_runs_valid_statuses_200(client: TestClient) -> None:
|
||
"""Все допустимые значения status проходят валидацию (ожидаем 200, не 422)."""
|
||
valid_statuses = ["done", "running", "banned", "zombie", "failed", "cancelled"]
|
||
for s in valid_statuses:
|
||
with patch("app.services.scrape_runs.list_all", return_value=(0, [])):
|
||
r = client.get("/api/v1/admin/scrape/runs", params={"status": s})
|
||
assert r.status_code == 200, f"status={s!r} должен возвращать 200, получили {r.status_code}"
|
||
|
||
|
||
# ── API 2: GET /scraper/health ───────────────────────────────────────────────
|
||
|
||
|
||
def test_health_assembles_browser_and_providers(client: TestClient) -> None:
|
||
"""browser /health + ipify пробинг → fetch_mode + browser + 3 провайдера."""
|
||
from app.api.v1 import admin as admin_module
|
||
|
||
async def fake_browser_health() -> admin_module.BrowserHealth:
|
||
return admin_module.BrowserHealth(
|
||
reachable=True,
|
||
browsers={"avito": False, "cian": True, "yandex": False, "generic": True},
|
||
)
|
||
|
||
async def fake_current_ip(proxy_url: str | None) -> str | None:
|
||
return "1.2.3.4" if proxy_url else None
|
||
|
||
proxy_urls = {
|
||
"avito": "http://u:p@host.a:10049",
|
||
"cian": None,
|
||
"yandex": "http://u:p@host.y:10051",
|
||
}
|
||
rotate_urls = {"avito": "http://ch/changeip", "cian": None, "yandex": None}
|
||
|
||
with (
|
||
patch.object(admin_module, "_probe_browser_health", fake_browser_health),
|
||
patch.object(admin_module, "_probe_current_ip", fake_current_ip),
|
||
patch.object(admin_module, "_provider_proxy_url", lambda s: proxy_urls[s]),
|
||
patch.object(admin_module, "_provider_rotate_url", lambda s: rotate_urls[s]),
|
||
patch.object(admin_module.settings, "scraper_fetch_mode", "browser"),
|
||
):
|
||
r = client.get("/api/v1/admin/scraper/health")
|
||
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["fetch_mode"] == "browser"
|
||
assert body["browser"]["reachable"] is True
|
||
assert body["browser"]["browsers"]["cian"] is True
|
||
assert body["browser"]["browsers"]["avito"] is False
|
||
|
||
by_source = {p["source"]: p for p in body["providers"]}
|
||
assert set(by_source) == {"avito", "cian", "yandex"}
|
||
assert by_source["avito"]["proxy_host"] == "host.a"
|
||
assert by_source["avito"]["proxy_port"] == 10049
|
||
assert by_source["avito"]["rotate_supported"] is True
|
||
assert by_source["avito"]["current_ip"] == "1.2.3.4"
|
||
# cian без proxy → host/port/ip null, rotate_supported False
|
||
assert by_source["cian"]["proxy_host"] is None
|
||
assert by_source["cian"]["current_ip"] is None
|
||
assert by_source["cian"]["rotate_supported"] is False
|
||
# yandex имеет proxy но нет rotate_url
|
||
assert by_source["yandex"]["proxy_host"] == "host.y"
|
||
assert by_source["yandex"]["rotate_supported"] is False
|
||
|
||
|
||
def test_health_browser_unreachable(client: TestClient) -> None:
|
||
"""httpx-ошибка на /health → reachable=False, browsers пуст."""
|
||
from app.api.v1 import admin as admin_module
|
||
|
||
async def fake_current_ip(proxy_url: str | None) -> str | None:
|
||
return None
|
||
|
||
class _BoomClient:
|
||
def __init__(self, *a: Any, **k: Any) -> None:
|
||
pass
|
||
|
||
async def __aenter__(self) -> _BoomClient:
|
||
return self
|
||
|
||
async def __aexit__(self, *a: Any) -> None:
|
||
return None
|
||
|
||
async def get(self, *a: Any, **k: Any) -> Any:
|
||
raise RuntimeError("browser down")
|
||
|
||
with (
|
||
patch.object(admin_module.httpx, "AsyncClient", _BoomClient),
|
||
patch.object(admin_module, "_probe_current_ip", fake_current_ip),
|
||
patch.object(admin_module, "_provider_proxy_url", lambda s: None),
|
||
patch.object(admin_module, "_provider_rotate_url", lambda s: None),
|
||
):
|
||
r = client.get("/api/v1/admin/scraper/health")
|
||
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["browser"]["reachable"] is False
|
||
assert body["browser"]["browsers"] == {}
|
||
|
||
|
||
# ── API 3: POST /scraper/{source}/rotate-ip ──────────────────────────────────
|
||
|
||
|
||
def test_rotate_ip_calls_changeip(client: TestClient) -> None:
|
||
"""rotate_url задан → GET changeip + format=json, возврат new_ip из ответа."""
|
||
from app.api.v1 import admin as admin_module
|
||
|
||
captured: dict[str, str] = {}
|
||
|
||
class _FakeResp:
|
||
def raise_for_status(self) -> None:
|
||
return None
|
||
|
||
def json(self) -> dict[str, str]:
|
||
return {"new_ip": "9.9.9.9"}
|
||
|
||
class _FakeClient:
|
||
def __init__(self, *a: Any, **k: Any) -> None:
|
||
pass
|
||
|
||
async def __aenter__(self) -> _FakeClient:
|
||
return self
|
||
|
||
async def __aexit__(self, *a: Any) -> None:
|
||
return None
|
||
|
||
async def get(self, url: str, *a: Any, **k: Any) -> _FakeResp:
|
||
captured["url"] = url
|
||
return _FakeResp()
|
||
|
||
with (
|
||
patch.object(admin_module.httpx, "AsyncClient", _FakeClient),
|
||
patch.object(admin_module.settings, "avito_proxy_rotate_url", "http://ch/changeip?key=x"),
|
||
):
|
||
r = client.post("/api/v1/admin/scraper/avito/rotate-ip")
|
||
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["ok"] is True
|
||
assert body["new_ip"] == "9.9.9.9"
|
||
assert "format=json" in captured["url"]
|
||
# уже был '?' в URL → должен использовать '&'
|
||
assert captured["url"].endswith("&format=json")
|
||
|
||
|
||
def test_rotate_ip_no_rotate_url(client: TestClient) -> None:
|
||
"""rotate_url не задан → ok=False + reason про auto-rotate, changeip НЕ дёргается."""
|
||
from app.api.v1 import admin as admin_module
|
||
|
||
with patch.object(admin_module.settings, "cian_proxy_rotate_url", None):
|
||
r = client.post("/api/v1/admin/scraper/cian/rotate-ip")
|
||
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["ok"] is False
|
||
assert body["new_ip"] is None
|
||
assert "auto-rotate" in body["reason"]
|
||
|
||
|
||
def test_rotate_ip_invalid_source_422(client: TestClient) -> None:
|
||
"""source вне {avito,cian,yandex} → 422 (Literal-валидация path)."""
|
||
r = client.post("/api/v1/admin/scraper/domclick/rotate-ip")
|
||
assert r.status_code == 422
|
||
|
||
|
||
def test_rotate_ip_changeip_error(client: TestClient) -> None:
|
||
"""httpx-ошибка changeip → ok=False + reason."""
|
||
from app.api.v1 import admin as admin_module
|
||
|
||
class _BoomClient:
|
||
def __init__(self, *a: Any, **k: Any) -> None:
|
||
pass
|
||
|
||
async def __aenter__(self) -> _BoomClient:
|
||
return self
|
||
|
||
async def __aexit__(self, *a: Any) -> None:
|
||
return None
|
||
|
||
async def get(self, *a: Any, **k: Any) -> Any:
|
||
raise RuntimeError("changeip 500")
|
||
|
||
with (
|
||
patch.object(admin_module.httpx, "AsyncClient", _BoomClient),
|
||
patch.object(admin_module.settings, "yandex_proxy_rotate_url", "http://ch/changeip"),
|
||
):
|
||
r = client.post("/api/v1/admin/scraper/yandex/rotate-ip")
|
||
|
||
assert r.status_code == 200
|
||
body = r.json()
|
||
assert body["ok"] is False
|
||
assert "changeip error" in body["reason"]
|