Merge pull request 'feat(scrapers): live pacing-регулятор + data-quality coverage API' (#1829) from feat/scrapers-config-apis into main
Some checks failed
Deploy Trade-In / build-backend (push) Blocked by required conditions
Deploy Trade-In / deploy (push) Blocked by required conditions
Deploy Trade-In / changes (push) Successful in 7s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been cancelled
Deploy Trade-In / test (push) Has been cancelled
Some checks failed
Deploy Trade-In / build-backend (push) Blocked by required conditions
Deploy Trade-In / deploy (push) Blocked by required conditions
Deploy Trade-In / changes (push) Successful in 7s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been cancelled
Deploy Trade-In / test (push) Has been cancelled
Reviewed-on: #1829
This commit is contained in:
commit
93783602c5
4 changed files with 655 additions and 2 deletions
|
|
@ -2110,3 +2110,203 @@ async def rotate_proxy_ip(
|
||||||
new_ip = data.get("new_ip") or data.get("ip") or data.get("proxy_ip")
|
new_ip = data.get("new_ip") or data.get("ip") or data.get("proxy_ip")
|
||||||
logger.info("rotate-ip: source=%s new_ip=%s", source, new_ip)
|
logger.info("rotate-ip: source=%s new_ip=%s", source, new_ip)
|
||||||
return RotateIpResponse(ok=True, new_ip=str(new_ip) if new_ip else None)
|
return RotateIpResponse(ok=True, new_ip=str(new_ip) if new_ip else None)
|
||||||
|
|
||||||
|
|
||||||
|
# ── Pacing live-регулятор (GET/PUT /scraper/pacing) ──────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class PacingProvider(BaseModel):
|
||||||
|
source: str
|
||||||
|
interval_s: float
|
||||||
|
env_default_s: float
|
||||||
|
|
||||||
|
|
||||||
|
class PacingResponse(BaseModel):
|
||||||
|
providers: list[PacingProvider]
|
||||||
|
|
||||||
|
|
||||||
|
class PacingUpdateRequest(BaseModel):
|
||||||
|
interval_s: float = Field(ge=0, le=120)
|
||||||
|
|
||||||
|
|
||||||
|
async def _proxy_browser_pacing_get() -> dict:
|
||||||
|
"""GET tradein-browser /pacing (timeout 5с). Пробрасывает ответ или кидает HTTPException."""
|
||||||
|
url = f"{settings.browser_http_endpoint.rstrip('/')}/pacing"
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||||
|
resp = await client.get(url)
|
||||||
|
resp.raise_for_status()
|
||||||
|
return resp.json() # type: ignore[no-any-return]
|
||||||
|
except httpx.HTTPStatusError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=502,
|
||||||
|
detail=f"browser /pacing returned {exc.response.status_code}",
|
||||||
|
) from exc
|
||||||
|
except Exception as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail=f"browser unreachable: {type(exc).__name__}",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/scraper/pacing", response_model=PacingResponse)
|
||||||
|
async def get_scraper_pacing() -> PacingResponse:
|
||||||
|
"""Текущие live-интервалы пейсинга из tradein-browser (in-memory, per-provider).
|
||||||
|
|
||||||
|
interval_s — живой интервал (может быть изменён PUT).
|
||||||
|
env_default_s — значение из env (к чему сбросится на рестарте).
|
||||||
|
502/503 при недоступности browser-сервиса.
|
||||||
|
"""
|
||||||
|
data = await _proxy_browser_pacing_get()
|
||||||
|
providers = [
|
||||||
|
PacingProvider(
|
||||||
|
source=p["source"],
|
||||||
|
interval_s=float(p["interval_s"]),
|
||||||
|
env_default_s=float(p["env_default_s"]),
|
||||||
|
)
|
||||||
|
for p in data.get("providers", [])
|
||||||
|
]
|
||||||
|
return PacingResponse(providers=providers)
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/scraper/pacing/{source}", response_model=dict)
|
||||||
|
async def update_scraper_pacing(
|
||||||
|
source: Literal["avito", "cian", "yandex", "generic"],
|
||||||
|
payload: PacingUpdateRequest,
|
||||||
|
) -> dict:
|
||||||
|
"""Обновить live-интервал пейсинга для провайдера в tradein-browser (in-memory).
|
||||||
|
|
||||||
|
Значение сбрасывается к env-дефолту при рестарте контейнера — by design.
|
||||||
|
interval_s: 0..120 (сек). Возвращает {ok, source, interval_s}.
|
||||||
|
502/503 при недоступности browser-сервиса.
|
||||||
|
"""
|
||||||
|
url = f"{settings.browser_http_endpoint.rstrip('/')}/pacing"
|
||||||
|
try:
|
||||||
|
async with httpx.AsyncClient(timeout=5.0) as client:
|
||||||
|
resp = await client.put(
|
||||||
|
url,
|
||||||
|
json={"source": source, "interval_s": payload.interval_s},
|
||||||
|
)
|
||||||
|
resp.raise_for_status()
|
||||||
|
return resp.json() # type: ignore[no-any-return]
|
||||||
|
except httpx.HTTPStatusError as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=502,
|
||||||
|
detail=f"browser /pacing returned {exc.response.status_code}",
|
||||||
|
) from exc
|
||||||
|
except Exception as exc:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=503,
|
||||||
|
detail=f"browser unreachable: {type(exc).__name__}",
|
||||||
|
) from exc
|
||||||
|
|
||||||
|
|
||||||
|
# ── Data-quality coverage (GET /scraper/data-quality) ────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class SourceCoverage(BaseModel):
|
||||||
|
source: str
|
||||||
|
active_count: int
|
||||||
|
fields: dict[str, float] # field_name -> fill% (0..100, round 1)
|
||||||
|
|
||||||
|
|
||||||
|
class HousesCoverage(BaseModel):
|
||||||
|
total: int
|
||||||
|
validated_pct: float # avito_validated_at IS NOT NULL %
|
||||||
|
rating_pct: float # rating_score IS NOT NULL %
|
||||||
|
house_type_pct: float # house_type IS NOT NULL %
|
||||||
|
reviews_count: int
|
||||||
|
|
||||||
|
|
||||||
|
class DataQualityResponse(BaseModel):
|
||||||
|
sources: list[SourceCoverage]
|
||||||
|
houses: HousesCoverage
|
||||||
|
|
||||||
|
|
||||||
|
# Поля listings для fill%-аудита. Каждый кортеж: (имя_поля, SQL-выражение IS NOT NULL).
|
||||||
|
# Для photo_urls отдельная логика (jsonb не NULL и не '[]').
|
||||||
|
_DQ_LISTING_FIELDS: list[tuple[str, str]] = [
|
||||||
|
("description", "description IS NOT NULL AND description <> ''"),
|
||||||
|
("photo_urls", "photo_urls IS NOT NULL AND photo_urls <> '[]'::jsonb"),
|
||||||
|
("address", "address IS NOT NULL AND address <> ''"),
|
||||||
|
("lat", "lat IS NOT NULL"),
|
||||||
|
("lon", "lon IS NOT NULL"),
|
||||||
|
("kitchen_area_m2", "kitchen_area_m2 IS NOT NULL"),
|
||||||
|
("living_area_m2", "living_area_m2 IS NOT NULL"),
|
||||||
|
("ceiling_height", "ceiling_height IS NOT NULL"),
|
||||||
|
("ceiling_height_m", "ceiling_height_m IS NOT NULL"),
|
||||||
|
("metro_stations", "metro_stations IS NOT NULL AND metro_stations <> '[]'::jsonb"),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/scraper/data-quality", response_model=DataQualityResponse)
|
||||||
|
def get_data_quality(
|
||||||
|
db: Annotated[Session, Depends(get_db)],
|
||||||
|
) -> DataQualityResponse:
|
||||||
|
"""Fill%-аудит detail-полей listings по source + дом-статистика.
|
||||||
|
|
||||||
|
Один проход per source через COUNT(*)...FILTER — не N запросов.
|
||||||
|
Поля listings: description, photo_urls, address, lat/lon, kitchen_area_m2,
|
||||||
|
living_area_m2, ceiling_height (cian), ceiling_height_m (avito), metro_stations.
|
||||||
|
houses: total, avito_validated_at%, rating_score%, house_type%.
|
||||||
|
house_reviews: общий count.
|
||||||
|
"""
|
||||||
|
# Строим single-pass SELECT для listings полей через FILTER-агрегаты.
|
||||||
|
# Структура: COUNT(*) FILTER (WHERE <expr>) / NULLIF(COUNT(*), 0) * 100
|
||||||
|
# Одним запросом получаем active_count + все fill-counts по каждому source.
|
||||||
|
filter_exprs = ", ".join(
|
||||||
|
f"COUNT(*) FILTER (WHERE {expr}) AS f_{name}" for name, expr in _DQ_LISTING_FIELDS
|
||||||
|
)
|
||||||
|
sql_listings = text(f"""
|
||||||
|
SELECT
|
||||||
|
source,
|
||||||
|
COUNT(*) AS active_count,
|
||||||
|
{filter_exprs}
|
||||||
|
FROM listings
|
||||||
|
WHERE is_active = true
|
||||||
|
GROUP BY source
|
||||||
|
ORDER BY source
|
||||||
|
""")
|
||||||
|
|
||||||
|
rows = db.execute(sql_listings).mappings().all()
|
||||||
|
|
||||||
|
sources: list[SourceCoverage] = []
|
||||||
|
for row in rows:
|
||||||
|
cnt = int(row["active_count"]) or 1 # защита от деления на ноль
|
||||||
|
fields: dict[str, float] = {}
|
||||||
|
for name, _ in _DQ_LISTING_FIELDS:
|
||||||
|
raw = row[f"f_{name}"]
|
||||||
|
fill_pct = round(float(raw or 0) / cnt * 100, 1)
|
||||||
|
fields[name] = fill_pct
|
||||||
|
sources.append(
|
||||||
|
SourceCoverage(
|
||||||
|
source=row["source"],
|
||||||
|
active_count=int(row["active_count"]),
|
||||||
|
fields=fields,
|
||||||
|
)
|
||||||
|
)
|
||||||
|
|
||||||
|
# Houses: один агрегат
|
||||||
|
sql_houses = text("""
|
||||||
|
SELECT
|
||||||
|
COUNT(*) AS total,
|
||||||
|
COUNT(*) FILTER (WHERE avito_validated_at IS NOT NULL) AS validated_cnt,
|
||||||
|
COUNT(*) FILTER (WHERE rating_score IS NOT NULL) AS rating_cnt,
|
||||||
|
COUNT(*) FILTER (WHERE house_type IS NOT NULL) AS house_type_cnt
|
||||||
|
FROM houses
|
||||||
|
""")
|
||||||
|
h = db.execute(sql_houses).mappings().one()
|
||||||
|
total_houses = int(h["total"]) or 1 # guard против деления на ноль в %
|
||||||
|
|
||||||
|
reviews_count_row = db.execute(text("SELECT COUNT(*) AS cnt FROM house_reviews")).scalar()
|
||||||
|
reviews_count = int(reviews_count_row or 0)
|
||||||
|
|
||||||
|
houses = HousesCoverage(
|
||||||
|
total=int(h["total"]),
|
||||||
|
validated_pct=round(float(h["validated_cnt"] or 0) / total_houses * 100, 1),
|
||||||
|
rating_pct=round(float(h["rating_cnt"] or 0) / total_houses * 100, 1),
|
||||||
|
house_type_pct=round(float(h["house_type_cnt"] or 0) / total_houses * 100, 1),
|
||||||
|
reviews_count=reviews_count,
|
||||||
|
)
|
||||||
|
|
||||||
|
return DataQualityResponse(sources=sources, houses=houses)
|
||||||
|
|
|
||||||
|
|
@ -306,3 +306,261 @@ def test_rotate_ip_changeip_error(client: TestClient) -> None:
|
||||||
body = r.json()
|
body = r.json()
|
||||||
assert body["ok"] is False
|
assert body["ok"] is False
|
||||||
assert "changeip error" in body["reason"]
|
assert "changeip error" in body["reason"]
|
||||||
|
|
||||||
|
|
||||||
|
# ── API 4: GET /scraper/pacing ───────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_get_proxies_browser_response(client: TestClient) -> None:
|
||||||
|
"""GET /scraper/pacing → парсит browser /pacing и возвращает PacingResponse."""
|
||||||
|
from app.api.v1 import admin as admin_module
|
||||||
|
|
||||||
|
fake_browser_resp = {
|
||||||
|
"providers": [
|
||||||
|
{"source": "avito", "interval_s": 3.0, "env_default_s": 2.0},
|
||||||
|
{"source": "cian", "interval_s": 18.0, "env_default_s": 2.0},
|
||||||
|
{"source": "yandex", "interval_s": 2.0, "env_default_s": 2.0},
|
||||||
|
{"source": "generic", "interval_s": 2.0, "env_default_s": 2.0},
|
||||||
|
]
|
||||||
|
}
|
||||||
|
|
||||||
|
async def _fake_proxy() -> dict:
|
||||||
|
return fake_browser_resp
|
||||||
|
|
||||||
|
with patch.object(admin_module, "_proxy_browser_pacing_get", _fake_proxy):
|
||||||
|
r = client.get("/api/v1/admin/scraper/pacing")
|
||||||
|
|
||||||
|
assert r.status_code == 200
|
||||||
|
body = r.json()
|
||||||
|
assert "providers" in body
|
||||||
|
by_source = {p["source"]: p for p in body["providers"]}
|
||||||
|
assert set(by_source) == {"avito", "cian", "yandex", "generic"}
|
||||||
|
assert by_source["cian"]["interval_s"] == 18.0
|
||||||
|
assert by_source["avito"]["env_default_s"] == 2.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_get_browser_unreachable_503(client: TestClient) -> None:
|
||||||
|
"""GET /scraper/pacing при недоступном browser → 503."""
|
||||||
|
from fastapi import HTTPException
|
||||||
|
|
||||||
|
from app.api.v1 import admin as admin_module
|
||||||
|
|
||||||
|
async def _boom() -> dict:
|
||||||
|
raise HTTPException(status_code=503, detail="browser unreachable: ConnectError")
|
||||||
|
|
||||||
|
with patch.object(admin_module, "_proxy_browser_pacing_get", _boom):
|
||||||
|
r = client.get("/api/v1/admin/scraper/pacing")
|
||||||
|
|
||||||
|
assert r.status_code == 503
|
||||||
|
|
||||||
|
|
||||||
|
# ── API 5: PUT /scraper/pacing/{source} ─────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_proxies_to_browser(client: TestClient) -> None:
|
||||||
|
"""PUT /scraper/pacing/avito → проксирует на browser и отдаёт {ok,source,interval_s}."""
|
||||||
|
from app.api.v1 import admin as admin_module
|
||||||
|
|
||||||
|
captured: dict[str, Any] = {}
|
||||||
|
|
||||||
|
class _FakeResp:
|
||||||
|
def raise_for_status(self) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def json(self) -> dict:
|
||||||
|
return {"ok": True, "source": "avito", "interval_s": 7.5}
|
||||||
|
|
||||||
|
class _FakeClient:
|
||||||
|
def __init__(self, *a: Any, **k: Any) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def __aenter__(self) -> _FakeClient:
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *a: Any) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def put(self, url: str, json: Any = None, **k: Any) -> _FakeResp:
|
||||||
|
captured["url"] = url
|
||||||
|
captured["json"] = json
|
||||||
|
return _FakeResp()
|
||||||
|
|
||||||
|
with patch.object(admin_module.httpx, "AsyncClient", _FakeClient):
|
||||||
|
r = client.put(
|
||||||
|
"/api/v1/admin/scraper/pacing/avito",
|
||||||
|
json={"interval_s": 7.5},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert r.status_code == 200
|
||||||
|
body = r.json()
|
||||||
|
assert body["ok"] is True
|
||||||
|
assert body["source"] == "avito"
|
||||||
|
assert body["interval_s"] == 7.5
|
||||||
|
assert captured["json"]["source"] == "avito"
|
||||||
|
assert captured["json"]["interval_s"] == 7.5
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_invalid_source_422(client: TestClient) -> None:
|
||||||
|
"""PUT /scraper/pacing/domclick → 422 (Literal-валидация)."""
|
||||||
|
r = client.put("/api/v1/admin/scraper/pacing/domclick", json={"interval_s": 5.0})
|
||||||
|
assert r.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_negative_interval_422(client: TestClient) -> None:
|
||||||
|
"""PUT /scraper/pacing/cian с interval_s < 0 → 422 (ge=0 валидация)."""
|
||||||
|
r = client.put("/api/v1/admin/scraper/pacing/cian", json={"interval_s": -1.0})
|
||||||
|
assert r.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_over_max_interval_422(client: TestClient) -> None:
|
||||||
|
"""PUT /scraper/pacing/yandex с interval_s > 120 → 422 (le=120 валидация)."""
|
||||||
|
r = client.put("/api/v1/admin/scraper/pacing/yandex", json={"interval_s": 200.0})
|
||||||
|
assert r.status_code == 422
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_browser_unreachable_503(client: TestClient) -> None:
|
||||||
|
"""PUT /scraper/pacing/avito при недоступном browser → 503."""
|
||||||
|
from app.api.v1 import admin as admin_module
|
||||||
|
|
||||||
|
class _BoomClient:
|
||||||
|
def __init__(self, *a: Any, **k: Any) -> None:
|
||||||
|
pass
|
||||||
|
|
||||||
|
async def __aenter__(self) -> _BoomClient:
|
||||||
|
return self
|
||||||
|
|
||||||
|
async def __aexit__(self, *a: Any) -> None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
async def put(self, *a: Any, **k: Any) -> Any:
|
||||||
|
raise RuntimeError("connection refused")
|
||||||
|
|
||||||
|
with patch.object(admin_module.httpx, "AsyncClient", _BoomClient):
|
||||||
|
r = client.put(
|
||||||
|
"/api/v1/admin/scraper/pacing/avito",
|
||||||
|
json={"interval_s": 5.0},
|
||||||
|
)
|
||||||
|
|
||||||
|
assert r.status_code == 503
|
||||||
|
|
||||||
|
|
||||||
|
# ── API 6: GET /scraper/data-quality ────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _make_dq_db_mock() -> MagicMock:
|
||||||
|
"""Мок db-сессии для GET /scraper/data-quality."""
|
||||||
|
db = MagicMock()
|
||||||
|
|
||||||
|
# listings rows: avito=1000 active, cian=500 active
|
||||||
|
avito_row = {
|
||||||
|
"source": "avito",
|
||||||
|
"active_count": 1000,
|
||||||
|
"f_description": 900,
|
||||||
|
"f_photo_urls": 980,
|
||||||
|
"f_address": 1000,
|
||||||
|
"f_lat": 950,
|
||||||
|
"f_lon": 950,
|
||||||
|
"f_kitchen_area_m2": 600,
|
||||||
|
"f_living_area_m2": 100,
|
||||||
|
"f_ceiling_height": 50,
|
||||||
|
"f_ceiling_height_m": 300,
|
||||||
|
"f_metro_stations": 700,
|
||||||
|
}
|
||||||
|
cian_row = {
|
||||||
|
"source": "cian",
|
||||||
|
"active_count": 500,
|
||||||
|
"f_description": 450,
|
||||||
|
"f_photo_urls": 490,
|
||||||
|
"f_address": 500,
|
||||||
|
"f_lat": 480,
|
||||||
|
"f_lon": 480,
|
||||||
|
"f_kitchen_area_m2": 400,
|
||||||
|
"f_living_area_m2": 400,
|
||||||
|
"f_ceiling_height": 350,
|
||||||
|
"f_ceiling_height_m": 0,
|
||||||
|
"f_metro_stations": 100,
|
||||||
|
}
|
||||||
|
|
||||||
|
houses_row = {
|
||||||
|
"total": 2000,
|
||||||
|
"validated_cnt": 1800,
|
||||||
|
"rating_cnt": 1200,
|
||||||
|
"house_type_cnt": 1900,
|
||||||
|
}
|
||||||
|
|
||||||
|
listing_mappings = MagicMock()
|
||||||
|
listing_mappings.all.return_value = [avito_row, cian_row]
|
||||||
|
|
||||||
|
houses_mappings = MagicMock()
|
||||||
|
houses_mappings.one.return_value = houses_row
|
||||||
|
|
||||||
|
reviews_scalar = 4500
|
||||||
|
|
||||||
|
# db.execute() вызывается 3 раза: listings SQL, houses SQL, house_reviews COUNT
|
||||||
|
execute_results = iter(
|
||||||
|
[
|
||||||
|
MagicMock(mappings=lambda: listing_mappings),
|
||||||
|
MagicMock(mappings=lambda: houses_mappings),
|
||||||
|
MagicMock(scalar=lambda: reviews_scalar),
|
||||||
|
]
|
||||||
|
)
|
||||||
|
db.execute.side_effect = lambda *a, **k: next(execute_results)
|
||||||
|
return db
|
||||||
|
|
||||||
|
|
||||||
|
def test_data_quality_shape(client: TestClient) -> None:
|
||||||
|
"""GET /scraper/data-quality → StructureCheck: sources + houses + reviews_count."""
|
||||||
|
from app.api.v1 import admin as admin_module # noqa: F401
|
||||||
|
from app.core.db import get_db
|
||||||
|
|
||||||
|
app = client.app
|
||||||
|
app.dependency_overrides[get_db] = lambda: _make_dq_db_mock()
|
||||||
|
|
||||||
|
r = client.get("/api/v1/admin/scraper/data-quality")
|
||||||
|
assert r.status_code == 200
|
||||||
|
body = r.json()
|
||||||
|
|
||||||
|
assert "sources" in body
|
||||||
|
assert "houses" in body
|
||||||
|
|
||||||
|
# two sources
|
||||||
|
sources = {s["source"]: s for s in body["sources"]}
|
||||||
|
assert set(sources) >= {"avito", "cian"}
|
||||||
|
|
||||||
|
avito = sources["avito"]
|
||||||
|
assert avito["active_count"] == 1000
|
||||||
|
fields = avito["fields"]
|
||||||
|
assert 0.0 <= fields["description"] <= 100.0
|
||||||
|
assert fields["description"] == pytest.approx(90.0, abs=0.1)
|
||||||
|
assert fields["photo_urls"] == pytest.approx(98.0, abs=0.1)
|
||||||
|
assert fields["kitchen_area_m2"] == pytest.approx(60.0, abs=0.1)
|
||||||
|
|
||||||
|
cian = sources["cian"]
|
||||||
|
assert cian["active_count"] == 500
|
||||||
|
assert cian["fields"]["ceiling_height"] == pytest.approx(70.0, abs=0.1)
|
||||||
|
assert cian["fields"]["ceiling_height_m"] == pytest.approx(0.0, abs=0.1)
|
||||||
|
|
||||||
|
houses = body["houses"]
|
||||||
|
assert houses["total"] == 2000
|
||||||
|
assert houses["validated_pct"] == pytest.approx(90.0, abs=0.1)
|
||||||
|
assert houses["rating_pct"] == pytest.approx(60.0, abs=0.1)
|
||||||
|
assert houses["house_type_pct"] == pytest.approx(95.0, abs=0.1)
|
||||||
|
assert houses["reviews_count"] == 4500
|
||||||
|
|
||||||
|
|
||||||
|
def test_data_quality_pct_in_range(client: TestClient) -> None:
|
||||||
|
"""Все fill% в [0,100]."""
|
||||||
|
from app.core.db import get_db
|
||||||
|
|
||||||
|
app = client.app
|
||||||
|
app.dependency_overrides[get_db] = lambda: _make_dq_db_mock()
|
||||||
|
|
||||||
|
r = client.get("/api/v1/admin/scraper/data-quality")
|
||||||
|
assert r.status_code == 200
|
||||||
|
body = r.json()
|
||||||
|
|
||||||
|
for src in body["sources"]:
|
||||||
|
for field_name, pct in src["fields"].items():
|
||||||
|
assert (
|
||||||
|
0.0 <= pct <= 100.0
|
||||||
|
), f"source={src['source']} field={field_name} pct={pct} вне [0,100]"
|
||||||
|
|
|
||||||
|
|
@ -710,6 +710,63 @@ class LoginError(Exception):
|
||||||
self.payload = payload
|
self.payload = payload
|
||||||
|
|
||||||
|
|
||||||
|
async def pacing_get_handler(request: web.Request) -> web.Response:
|
||||||
|
"""GET /pacing → текущие live-интервалы + env-дефолты для всех провайдеров.
|
||||||
|
|
||||||
|
interval_s — живое значение из _MIN_PAGE_INTERVAL_BY_PROVIDER (изменяется PUT).
|
||||||
|
env_default_s — что вернёт _resolve_min_interval(p) из os.environ (reset при рестарте).
|
||||||
|
Всегда 200; не требует browser-инстансов (memory-only).
|
||||||
|
"""
|
||||||
|
providers = []
|
||||||
|
for p in PROVIDERS:
|
||||||
|
providers.append({
|
||||||
|
"source": p,
|
||||||
|
"interval_s": _MIN_PAGE_INTERVAL_BY_PROVIDER.get(p, BROWSER_MIN_PAGE_INTERVAL_S),
|
||||||
|
"env_default_s": _resolve_min_interval(p),
|
||||||
|
})
|
||||||
|
return web.json_response({"providers": providers})
|
||||||
|
|
||||||
|
|
||||||
|
async def pacing_put_handler(request: web.Request) -> web.Response:
|
||||||
|
"""PUT /pacing {"source": <p>, "interval_s": <float>} → обновляет in-memory интервал.
|
||||||
|
|
||||||
|
Валидация: source ∈ {avito,cian,yandex,generic}, interval_s ≥ 0.
|
||||||
|
Сброс к env-дефолту происходит при рестарте контейнера (by design).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
body = await request.json()
|
||||||
|
except Exception:
|
||||||
|
return web.json_response({"error": "invalid JSON body"}, status=400)
|
||||||
|
|
||||||
|
source = body.get("source")
|
||||||
|
if not isinstance(source, str) or source not in PROVIDERS:
|
||||||
|
return web.json_response(
|
||||||
|
{"error": f"source must be one of {list(PROVIDERS)}"},
|
||||||
|
status=400,
|
||||||
|
)
|
||||||
|
|
||||||
|
interval_s = body.get("interval_s")
|
||||||
|
if not isinstance(interval_s, (int, float)):
|
||||||
|
return web.json_response(
|
||||||
|
{"error": "interval_s must be a non-negative number"},
|
||||||
|
status=422,
|
||||||
|
)
|
||||||
|
interval_s = float(interval_s)
|
||||||
|
if interval_s < 0:
|
||||||
|
return web.json_response(
|
||||||
|
{"error": "interval_s must be >= 0"},
|
||||||
|
status=422,
|
||||||
|
)
|
||||||
|
|
||||||
|
_MIN_PAGE_INTERVAL_BY_PROVIDER[source] = interval_s
|
||||||
|
logger.info(
|
||||||
|
"tradein-browser: pacing updated source=%s interval_s=%.2f (resets on restart)",
|
||||||
|
source,
|
||||||
|
interval_s,
|
||||||
|
)
|
||||||
|
return web.json_response({"ok": True, "source": source, "interval_s": interval_s})
|
||||||
|
|
||||||
|
|
||||||
async def login_handler(request: web.Request) -> web.Response:
|
async def login_handler(request: web.Request) -> web.Response:
|
||||||
"""POST /login {...} → {"cookies": [...]}
|
"""POST /login {...} → {"cookies": [...]}
|
||||||
|
|
||||||
|
|
@ -919,6 +976,8 @@ def build_app() -> web.Application:
|
||||||
app.router.add_get("/health", health_handler)
|
app.router.add_get("/health", health_handler)
|
||||||
app.router.add_post("/fetch", fetch_handler)
|
app.router.add_post("/fetch", fetch_handler)
|
||||||
app.router.add_post("/login", login_handler)
|
app.router.add_post("/login", login_handler)
|
||||||
|
app.router.add_get("/pacing", pacing_get_handler)
|
||||||
|
app.router.add_put("/pacing", pacing_put_handler)
|
||||||
return app
|
return app
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,12 +1,14 @@
|
||||||
"""test_server_pacing.py — юниты для per-provider pacing-интервалов (#1812 follow-up).
|
"""test_server_pacing.py — юниты для per-provider pacing-интервалов + HTTP-эндпоинтов.
|
||||||
|
|
||||||
Проверяет:
|
Проверяет:
|
||||||
1. per-provider env override переопределяет глобал для конкретного провайдера;
|
1. per-provider env override переопределяет глобал для конкретного провайдера;
|
||||||
2. отсутствие per-provider env → фолбэк на глобальный BROWSER_MIN_PAGE_INTERVAL_S;
|
2. отсутствие per-provider env → фолбэк на глобальный BROWSER_MIN_PAGE_INTERVAL_S;
|
||||||
3. невалидный per-provider env (не float) → фолбэк на глобал, без исключения;
|
3. невалидный per-provider env (не float) → фолбэк на глобал, без исключения;
|
||||||
4. пустая строка как per-provider env → фолбэк на глобал (ValueError при float("")).
|
4. пустая строка как per-provider env → фолбэк на глобал (ValueError при float("")).
|
||||||
|
5. GET /pacing — возвращает текущие live-значения + env-дефолты для всех провайдеров;
|
||||||
|
6. PUT /pacing — обновляет _MIN_PAGE_INTERVAL_BY_PROVIDER, валидирует source/interval_s.
|
||||||
|
|
||||||
camoufox/Playwright НЕ поднимается: _resolve_min_interval вызывается с фейковым environ.
|
camoufox/Playwright НЕ поднимается: тесты работают с чистым dict'ом и aiohttp mock.
|
||||||
|
|
||||||
Запуск (из tradein-mvp/browser/)::
|
Запуск (из tradein-mvp/browser/)::
|
||||||
|
|
||||||
|
|
@ -15,8 +17,14 @@ camoufox/Playwright НЕ поднимается: _resolve_min_interval вызы
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
import importlib.util
|
import importlib.util
|
||||||
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
from typing import Any
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
from aiohttp.test_utils import make_mocked_request
|
||||||
|
|
||||||
# server.py — не пакет (отдельный сервис без __init__/pyproject). Грузим по пути.
|
# server.py — не пакет (отдельный сервис без __init__/pyproject). Грузим по пути.
|
||||||
_SERVER_PATH = Path(__file__).resolve().parent / "server.py"
|
_SERVER_PATH = Path(__file__).resolve().parent / "server.py"
|
||||||
|
|
@ -102,3 +110,131 @@ def test_all_providers_covered() -> None:
|
||||||
for provider in ("avito", "cian", "yandex", "generic"):
|
for provider in ("avito", "cian", "yandex", "generic"):
|
||||||
result = server._resolve_min_interval(provider, {})
|
result = server._resolve_min_interval(provider, {})
|
||||||
assert isinstance(result, float), f"{provider}: ожидали float, получили {type(result)}"
|
assert isinstance(result, float), f"{provider}: ожидали float, получили {type(result)}"
|
||||||
|
|
||||||
|
|
||||||
|
# ── GET /pacing ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _json_body(response: Any) -> dict[str, Any]:
|
||||||
|
return json.loads(response.body.decode())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _reset_pacing_state(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""Восстанавливаем _MIN_PAGE_INTERVAL_BY_PROVIDER после каждого теста."""
|
||||||
|
original = dict(server._MIN_PAGE_INTERVAL_BY_PROVIDER)
|
||||||
|
yield
|
||||||
|
server._MIN_PAGE_INTERVAL_BY_PROVIDER.clear()
|
||||||
|
server._MIN_PAGE_INTERVAL_BY_PROVIDER.update(original)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_get_returns_all_providers() -> None:
|
||||||
|
"""GET /pacing возвращает providers для всех 4 источников."""
|
||||||
|
response = asyncio.run(server.pacing_get_handler(make_mocked_request("GET", "/pacing")))
|
||||||
|
body = _json_body(response)
|
||||||
|
assert response.status == 200
|
||||||
|
providers = {p["source"]: p for p in body["providers"]}
|
||||||
|
assert set(providers) == {"avito", "cian", "yandex", "generic"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_get_reflects_live_value(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||||
|
"""GET /pacing.interval_s отражает живое значение из _MIN_PAGE_INTERVAL_BY_PROVIDER."""
|
||||||
|
server._MIN_PAGE_INTERVAL_BY_PROVIDER["cian"] = 99.0
|
||||||
|
response = asyncio.run(server.pacing_get_handler(make_mocked_request("GET", "/pacing")))
|
||||||
|
body = _json_body(response)
|
||||||
|
providers = {p["source"]: p for p in body["providers"]}
|
||||||
|
assert providers["cian"]["interval_s"] == 99.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_get_has_env_default() -> None:
|
||||||
|
"""GET /pacing.env_default_s — это _resolve_min_interval(provider) из os.environ."""
|
||||||
|
response = asyncio.run(server.pacing_get_handler(make_mocked_request("GET", "/pacing")))
|
||||||
|
body = _json_body(response)
|
||||||
|
for p in body["providers"]:
|
||||||
|
assert isinstance(p["env_default_s"], float), (
|
||||||
|
f"{p['source']}: env_default_s должен быть float"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# ── PUT /pacing ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _put_request(body_dict: dict) -> Any:
|
||||||
|
req = make_mocked_request("PUT", "/pacing")
|
||||||
|
req.json = lambda: _coro_value(body_dict) # type: ignore[method-assign]
|
||||||
|
return req
|
||||||
|
|
||||||
|
|
||||||
|
def _coro_value(value: Any) -> Any:
|
||||||
|
async def _inner() -> Any:
|
||||||
|
return value
|
||||||
|
return _inner()
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_updates_dict() -> None:
|
||||||
|
"""PUT /pacing с валидным source+interval_s обновляет _MIN_PAGE_INTERVAL_BY_PROVIDER."""
|
||||||
|
original = server._MIN_PAGE_INTERVAL_BY_PROVIDER.get("avito")
|
||||||
|
asyncio.run(server.pacing_put_handler(_put_request({"source": "avito", "interval_s": 42.0})))
|
||||||
|
assert server._MIN_PAGE_INTERVAL_BY_PROVIDER["avito"] == 42.0
|
||||||
|
# Восстановить (на случай если фикстура autouse пропустит)
|
||||||
|
if original is not None:
|
||||||
|
server._MIN_PAGE_INTERVAL_BY_PROVIDER["avito"] = original
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_returns_ok() -> None:
|
||||||
|
"""PUT /pacing возвращает {ok:true, source, interval_s}."""
|
||||||
|
response = asyncio.run(
|
||||||
|
server.pacing_put_handler(_put_request({"source": "yandex", "interval_s": 15.5}))
|
||||||
|
)
|
||||||
|
assert response.status == 200
|
||||||
|
body = _json_body(response)
|
||||||
|
assert body["ok"] is True
|
||||||
|
assert body["source"] == "yandex"
|
||||||
|
assert body["interval_s"] == 15.5
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_zero_disables_pacing() -> None:
|
||||||
|
"""PUT /pacing с interval_s=0 разрешён (выключение пейсинга)."""
|
||||||
|
response = asyncio.run(
|
||||||
|
server.pacing_put_handler(_put_request({"source": "cian", "interval_s": 0}))
|
||||||
|
)
|
||||||
|
assert response.status == 200
|
||||||
|
assert server._MIN_PAGE_INTERVAL_BY_PROVIDER["cian"] == 0.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_bad_source_400() -> None:
|
||||||
|
"""PUT /pacing с невалидным source → 400."""
|
||||||
|
response = asyncio.run(
|
||||||
|
server.pacing_put_handler(_put_request({"source": "domclick", "interval_s": 5.0}))
|
||||||
|
)
|
||||||
|
assert response.status == 400
|
||||||
|
body = _json_body(response)
|
||||||
|
assert "source" in body["error"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_negative_interval_422() -> None:
|
||||||
|
"""PUT /pacing с interval_s < 0 → 422."""
|
||||||
|
response = asyncio.run(
|
||||||
|
server.pacing_put_handler(_put_request({"source": "avito", "interval_s": -1.0}))
|
||||||
|
)
|
||||||
|
assert response.status == 422
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_missing_interval_422() -> None:
|
||||||
|
"""PUT /pacing без interval_s → 422 (не число)."""
|
||||||
|
response = asyncio.run(
|
||||||
|
server.pacing_put_handler(_put_request({"source": "avito", "interval_s": "fast"}))
|
||||||
|
)
|
||||||
|
assert response.status == 422
|
||||||
|
|
||||||
|
|
||||||
|
def test_pacing_put_invalid_json_400() -> None:
|
||||||
|
"""PUT /pacing с битым JSON → 400."""
|
||||||
|
req = make_mocked_request("PUT", "/pacing")
|
||||||
|
|
||||||
|
async def _boom() -> Any:
|
||||||
|
raise ValueError("not json")
|
||||||
|
|
||||||
|
req.json = _boom # type: ignore[method-assign]
|
||||||
|
response = asyncio.run(server.pacing_put_handler(req))
|
||||||
|
assert response.status == 400
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue