bot-backend
b6634ea40c
- browser/server.py: GET /pacing и PUT /pacing — read/write _MIN_PAGE_INTERVAL_BY_PROVIDER
in-memory; валидация source in PROVIDERS, interval_s >= 0; логирует изменения; сбрасывается
к env-дефолту на рестарте by design
- backend/admin.py: GET /scraper/pacing — прокси к browser /pacing, PacingResponse pydantic;
PUT /scraper/pacing/{source} — прокси PUT с валидацией ge=0 le=120; 502/503 при недоступности
- backend/admin.py: GET /scraper/data-quality — single-pass FILTER-агрегаты по listings WHERE
is_active GROUP BY source (description/photo_urls/address/lat/lon/kitchen_area_m2/living_area_m2/
ceiling_height/ceiling_height_m/metro_stations); houses (total/avito_validated_at%/rating_score%/
house_type%); house_reviews count
- browser/test_server_pacing.py: 10 новых тестов GET+PUT /pacing (16 total, все зелёные)
- backend/tests/test_scraper_admin_apis.py: тесты pacing-прокси + data-quality shape/pct-range
(21 total, все зелёные)
240 lines
10 KiB
Python
240 lines
10 KiB
Python
"""test_server_pacing.py — юниты для per-provider pacing-интервалов + HTTP-эндпоинтов.
|
||
|
||
Проверяет:
|
||
1. per-provider env override переопределяет глобал для конкретного провайдера;
|
||
2. отсутствие per-provider env → фолбэк на глобальный BROWSER_MIN_PAGE_INTERVAL_S;
|
||
3. невалидный per-provider env (не float) → фолбэк на глобал, без исключения;
|
||
4. пустая строка как per-provider env → фолбэк на глобал (ValueError при float("")).
|
||
5. GET /pacing — возвращает текущие live-значения + env-дефолты для всех провайдеров;
|
||
6. PUT /pacing — обновляет _MIN_PAGE_INTERVAL_BY_PROVIDER, валидирует source/interval_s.
|
||
|
||
camoufox/Playwright НЕ поднимается: тесты работают с чистым dict'ом и aiohttp mock.
|
||
|
||
Запуск (из tradein-mvp/browser/)::
|
||
|
||
python -m pytest test_server_pacing.py -q
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import importlib.util
|
||
import json
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
from aiohttp.test_utils import make_mocked_request
|
||
|
||
# server.py — не пакет (отдельный сервис без __init__/pyproject). Грузим по пути.
|
||
_SERVER_PATH = Path(__file__).resolve().parent / "server.py"
|
||
_spec = importlib.util.spec_from_file_location("tradein_browser_server", _SERVER_PATH)
|
||
assert _spec is not None and _spec.loader is not None
|
||
server = importlib.util.module_from_spec(_spec)
|
||
_spec.loader.exec_module(server)
|
||
|
||
|
||
# ── _resolve_min_interval ────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_per_provider_override_replaces_global() -> None:
|
||
"""BROWSER_MIN_PAGE_INTERVAL_S_CIAN=18 + глобал=12 → cian=18, avito=12."""
|
||
fake_env = {
|
||
"BROWSER_MIN_PAGE_INTERVAL_S": "12",
|
||
"BROWSER_MIN_PAGE_INTERVAL_S_CIAN": "18",
|
||
}
|
||
# Патчим глобал в модуле, чтобы фолбэк брался из него (не из os.environ при load).
|
||
original_global = server.BROWSER_MIN_PAGE_INTERVAL_S
|
||
server.BROWSER_MIN_PAGE_INTERVAL_S = 12.0
|
||
try:
|
||
assert server._resolve_min_interval("cian", fake_env) == 18.0
|
||
assert server._resolve_min_interval("avito", fake_env) == 12.0
|
||
finally:
|
||
server.BROWSER_MIN_PAGE_INTERVAL_S = original_global
|
||
|
||
|
||
def test_no_per_provider_env_falls_back_to_global() -> None:
|
||
"""Нет BROWSER_MIN_PAGE_INTERVAL_S_YANDEX → фолбэк на глобал."""
|
||
fake_env: dict[str, str] = {}
|
||
original_global = server.BROWSER_MIN_PAGE_INTERVAL_S
|
||
server.BROWSER_MIN_PAGE_INTERVAL_S = 5.0
|
||
try:
|
||
result = server._resolve_min_interval("yandex", fake_env)
|
||
assert result == 5.0
|
||
finally:
|
||
server.BROWSER_MIN_PAGE_INTERVAL_S = original_global
|
||
|
||
|
||
def test_invalid_per_provider_env_falls_back_to_global() -> None:
|
||
"""Невалидный env (не float) → фолбэк на глобал, без исключения."""
|
||
fake_env = {"BROWSER_MIN_PAGE_INTERVAL_S_AVITO": "abc"}
|
||
original_global = server.BROWSER_MIN_PAGE_INTERVAL_S
|
||
server.BROWSER_MIN_PAGE_INTERVAL_S = 7.0
|
||
try:
|
||
result = server._resolve_min_interval("avito", fake_env)
|
||
assert result == 7.0
|
||
finally:
|
||
server.BROWSER_MIN_PAGE_INTERVAL_S = original_global
|
||
|
||
|
||
def test_empty_string_per_provider_env_falls_back_to_global() -> None:
|
||
"""Пустая строка env → float("") → ValueError → фолбэк на глобал."""
|
||
fake_env = {"BROWSER_MIN_PAGE_INTERVAL_S_GENERIC": ""}
|
||
original_global = server.BROWSER_MIN_PAGE_INTERVAL_S
|
||
server.BROWSER_MIN_PAGE_INTERVAL_S = 3.0
|
||
try:
|
||
result = server._resolve_min_interval("generic", fake_env)
|
||
assert result == 3.0
|
||
finally:
|
||
server.BROWSER_MIN_PAGE_INTERVAL_S = original_global
|
||
|
||
|
||
def test_zero_per_provider_env_disables_pacing() -> None:
|
||
"""BROWSER_MIN_PAGE_INTERVAL_S_AVITO=0 → 0.0 (без пейсинга для avito)."""
|
||
fake_env = {
|
||
"BROWSER_MIN_PAGE_INTERVAL_S": "12",
|
||
"BROWSER_MIN_PAGE_INTERVAL_S_AVITO": "0",
|
||
}
|
||
original_global = server.BROWSER_MIN_PAGE_INTERVAL_S
|
||
server.BROWSER_MIN_PAGE_INTERVAL_S = 12.0
|
||
try:
|
||
assert server._resolve_min_interval("avito", fake_env) == 0.0
|
||
# Другие провайдеры без override получают глобал.
|
||
assert server._resolve_min_interval("cian", fake_env) == 12.0
|
||
finally:
|
||
server.BROWSER_MIN_PAGE_INTERVAL_S = original_global
|
||
|
||
|
||
def test_all_providers_covered() -> None:
|
||
"""Все 4 провайдера (avito/cian/yandex/generic) обрабатываются резолвером."""
|
||
for provider in ("avito", "cian", "yandex", "generic"):
|
||
result = server._resolve_min_interval(provider, {})
|
||
assert isinstance(result, float), f"{provider}: ожидали float, получили {type(result)}"
|
||
|
||
|
||
# ── GET /pacing ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _json_body(response: Any) -> dict[str, Any]:
|
||
return json.loads(response.body.decode())
|
||
|
||
|
||
@pytest.fixture(autouse=True)
|
||
def _reset_pacing_state(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""Восстанавливаем _MIN_PAGE_INTERVAL_BY_PROVIDER после каждого теста."""
|
||
original = dict(server._MIN_PAGE_INTERVAL_BY_PROVIDER)
|
||
yield
|
||
server._MIN_PAGE_INTERVAL_BY_PROVIDER.clear()
|
||
server._MIN_PAGE_INTERVAL_BY_PROVIDER.update(original)
|
||
|
||
|
||
def test_pacing_get_returns_all_providers() -> None:
|
||
"""GET /pacing возвращает providers для всех 4 источников."""
|
||
response = asyncio.run(server.pacing_get_handler(make_mocked_request("GET", "/pacing")))
|
||
body = _json_body(response)
|
||
assert response.status == 200
|
||
providers = {p["source"]: p for p in body["providers"]}
|
||
assert set(providers) == {"avito", "cian", "yandex", "generic"}
|
||
|
||
|
||
def test_pacing_get_reflects_live_value(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""GET /pacing.interval_s отражает живое значение из _MIN_PAGE_INTERVAL_BY_PROVIDER."""
|
||
server._MIN_PAGE_INTERVAL_BY_PROVIDER["cian"] = 99.0
|
||
response = asyncio.run(server.pacing_get_handler(make_mocked_request("GET", "/pacing")))
|
||
body = _json_body(response)
|
||
providers = {p["source"]: p for p in body["providers"]}
|
||
assert providers["cian"]["interval_s"] == 99.0
|
||
|
||
|
||
def test_pacing_get_has_env_default() -> None:
|
||
"""GET /pacing.env_default_s — это _resolve_min_interval(provider) из os.environ."""
|
||
response = asyncio.run(server.pacing_get_handler(make_mocked_request("GET", "/pacing")))
|
||
body = _json_body(response)
|
||
for p in body["providers"]:
|
||
assert isinstance(p["env_default_s"], float), (
|
||
f"{p['source']}: env_default_s должен быть float"
|
||
)
|
||
|
||
|
||
# ── PUT /pacing ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def _put_request(body_dict: dict) -> Any:
|
||
req = make_mocked_request("PUT", "/pacing")
|
||
req.json = lambda: _coro_value(body_dict) # type: ignore[method-assign]
|
||
return req
|
||
|
||
|
||
def _coro_value(value: Any) -> Any:
|
||
async def _inner() -> Any:
|
||
return value
|
||
return _inner()
|
||
|
||
|
||
def test_pacing_put_updates_dict() -> None:
|
||
"""PUT /pacing с валидным source+interval_s обновляет _MIN_PAGE_INTERVAL_BY_PROVIDER."""
|
||
original = server._MIN_PAGE_INTERVAL_BY_PROVIDER.get("avito")
|
||
asyncio.run(server.pacing_put_handler(_put_request({"source": "avito", "interval_s": 42.0})))
|
||
assert server._MIN_PAGE_INTERVAL_BY_PROVIDER["avito"] == 42.0
|
||
# Восстановить (на случай если фикстура autouse пропустит)
|
||
if original is not None:
|
||
server._MIN_PAGE_INTERVAL_BY_PROVIDER["avito"] = original
|
||
|
||
|
||
def test_pacing_put_returns_ok() -> None:
|
||
"""PUT /pacing возвращает {ok:true, source, interval_s}."""
|
||
response = asyncio.run(
|
||
server.pacing_put_handler(_put_request({"source": "yandex", "interval_s": 15.5}))
|
||
)
|
||
assert response.status == 200
|
||
body = _json_body(response)
|
||
assert body["ok"] is True
|
||
assert body["source"] == "yandex"
|
||
assert body["interval_s"] == 15.5
|
||
|
||
|
||
def test_pacing_put_zero_disables_pacing() -> None:
|
||
"""PUT /pacing с interval_s=0 разрешён (выключение пейсинга)."""
|
||
response = asyncio.run(
|
||
server.pacing_put_handler(_put_request({"source": "cian", "interval_s": 0}))
|
||
)
|
||
assert response.status == 200
|
||
assert server._MIN_PAGE_INTERVAL_BY_PROVIDER["cian"] == 0.0
|
||
|
||
|
||
def test_pacing_put_bad_source_400() -> None:
|
||
"""PUT /pacing с невалидным source → 400."""
|
||
response = asyncio.run(
|
||
server.pacing_put_handler(_put_request({"source": "domclick", "interval_s": 5.0}))
|
||
)
|
||
assert response.status == 400
|
||
body = _json_body(response)
|
||
assert "source" in body["error"]
|
||
|
||
|
||
def test_pacing_put_negative_interval_422() -> None:
|
||
"""PUT /pacing с interval_s < 0 → 422."""
|
||
response = asyncio.run(
|
||
server.pacing_put_handler(_put_request({"source": "avito", "interval_s": -1.0}))
|
||
)
|
||
assert response.status == 422
|
||
|
||
|
||
def test_pacing_put_missing_interval_422() -> None:
|
||
"""PUT /pacing без interval_s → 422 (не число)."""
|
||
response = asyncio.run(
|
||
server.pacing_put_handler(_put_request({"source": "avito", "interval_s": "fast"}))
|
||
)
|
||
assert response.status == 422
|
||
|
||
|
||
def test_pacing_put_invalid_json_400() -> None:
|
||
"""PUT /pacing с битым JSON → 400."""
|
||
req = make_mocked_request("PUT", "/pacing")
|
||
|
||
async def _boom() -> Any:
|
||
raise ValueError("not json")
|
||
|
||
req.json = _boom # type: ignore[method-assign]
|
||
response = asyncio.run(server.pacing_put_handler(req))
|
||
assert response.status == 400
|