Light1YT
1ea8d0b721
kn (DOM.РФ ЖК) is scheduled WEEKLY (settings.scrape_kn_cron='15 4 * * mon'), but its freshness thresholds were fresh_days=2 / stale_days=5 — so the monitor flagged kn "stale" every Wed-Sun by design, a guaranteed weekly false-positive that would desensitize the team to the GlitchTip alert. The freshness monitor (#1904) caught this on its first prod run (06-26 09:00 MSK: "kn [stale, 3.9d]"). Recalibrate to weekly cadence (mirror objective 7/14): fresh_days=8 (full week + Monday-run completion buffer), stale_days=14 (alert only after ~two missed Mondays). +regression test pinning kn stays fresh through the week. Refs #73
210 lines
7.9 KiB
Python
210 lines
7.9 KiB
Python
"""Тесты для GET /api/v1/admin/scrape/freshness (#73 B5-3).
|
||
|
||
Эндпоинт читает 5 run-таблиц (kn / objective / nspd / nspd_geo / cadastre) и
|
||
для каждой считает свежесть. Здесь DB замокан: на каждый per-source SELECT
|
||
возвращаем заранее заданную строку, чтобы проверить классификацию
|
||
fresh/stale/critical, маппинг status (ok/stale/failed) и агрегат overall_status
|
||
по критичным источникам.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import UTC, datetime, timedelta
|
||
from typing import Any
|
||
from unittest.mock import MagicMock
|
||
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.api.v1 import admin_scrape
|
||
from app.core.db import get_db
|
||
from app.main import app
|
||
|
||
ENDPOINT = "/api/v1/admin/scrape/freshness"
|
||
|
||
|
||
def _ago(days: float) -> datetime:
|
||
return datetime.now(UTC) - timedelta(days=days)
|
||
|
||
|
||
def _make_db(per_source: dict[str, dict[str, Any]]) -> MagicMock:
|
||
"""Db-mock: каждый execute()'s SELECT относится к одной run-таблице; матчим по
|
||
имени таблицы в SQL и возвращаем соответствующую заранее заданную строку.
|
||
|
||
per_source: ключ — имя run-таблицы, значение — mapping-строка (как .first()).
|
||
"""
|
||
db = MagicMock()
|
||
|
||
def execute(stmt: Any, params: Any = None) -> Any:
|
||
sql = str(stmt)
|
||
matched: dict[str, Any] | None = None
|
||
# Матчим по FROM <table> — самый длинный матч (nspd_geo_jobs vs nspd_*).
|
||
for table in sorted(per_source, key=len, reverse=True):
|
||
if f"FROM {table}" in sql:
|
||
matched = per_source[table]
|
||
break
|
||
result = MagicMock()
|
||
result.mappings.return_value.first.return_value = matched
|
||
return result
|
||
|
||
db.execute = execute
|
||
return db
|
||
|
||
|
||
def _override_db(db: MagicMock) -> None:
|
||
def _factory():
|
||
yield db
|
||
|
||
app.dependency_overrides[get_db] = _factory
|
||
|
||
|
||
def test_freshness_mixed_statuses() -> None:
|
||
"""kn свежий (ok), objective stale, nspd без успехов (failed), nspd_geo critical,
|
||
cadastre fresh. overall_status = failed (по критичному nspd_geo→failed)."""
|
||
rows = {
|
||
"kn_scrape_runs": {
|
||
"last_success_at": _ago(1.0), # < fresh_days=2 → fresh → ok
|
||
"last_attempt_at": _ago(1.0),
|
||
"upd_24h": 1500,
|
||
"upd_7d": 1500,
|
||
"last_status": "done",
|
||
},
|
||
"objective_scrape_runs": {
|
||
"last_success_at": _ago(10.0), # fresh<7, stale<14 → stale
|
||
"last_attempt_at": _ago(10.0),
|
||
"upd_24h": 0,
|
||
"upd_7d": 325000,
|
||
"last_status": "done",
|
||
},
|
||
"nspd_scrape_runs": {
|
||
"last_success_at": None, # нет успехов → failed
|
||
"last_attempt_at": _ago(20.0),
|
||
"upd_24h": 0,
|
||
"upd_7d": 0,
|
||
"last_status": "failed",
|
||
},
|
||
"nspd_geo_jobs": {
|
||
"last_success_at": _ago(40.0), # >= stale_days=30 → critical → failed
|
||
"last_attempt_at": _ago(40.0),
|
||
"upd_24h": 0,
|
||
"upd_7d": 0,
|
||
"last_status": "done",
|
||
},
|
||
"cadastre_jobs": {
|
||
"last_success_at": _ago(3.0), # < fresh_days=14 → fresh → ok
|
||
"last_attempt_at": _ago(3.0),
|
||
"upd_24h": 5,
|
||
"upd_7d": 5,
|
||
"last_status": "done",
|
||
},
|
||
}
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
body = resp.json()
|
||
assert "generated_at" in body
|
||
assert len(body["sources"]) == 5
|
||
by_src = {s["source"]: s for s in body["sources"]}
|
||
|
||
assert by_src["kn"]["status"] == "ok"
|
||
assert by_src["kn"]["freshness"] == "fresh"
|
||
assert by_src["kn"]["objects_updated_7d"] == 1500
|
||
assert by_src["kn"]["critical"] is True
|
||
|
||
assert by_src["objective"]["status"] == "stale"
|
||
assert by_src["objective"]["freshness"] == "stale"
|
||
|
||
assert by_src["nspd"]["status"] == "failed"
|
||
assert by_src["nspd"]["last_success_at"] is None
|
||
assert by_src["nspd"]["age_days"] is None
|
||
|
||
assert by_src["nspd_geo"]["status"] == "failed"
|
||
assert by_src["nspd_geo"]["freshness"] == "critical"
|
||
|
||
assert by_src["cadastre"]["status"] == "ok"
|
||
|
||
# overall_status — худший по критичным (kn=ok, objective=stale) → stale.
|
||
# nspd_geo критичным НЕ помечен, поэтому его failed в overall не попадает.
|
||
assert by_src["nspd_geo"]["critical"] is False
|
||
assert body["overall_status"] == "stale"
|
||
|
||
|
||
def test_freshness_all_healthy() -> None:
|
||
"""Все источники свежие → overall_status=ok."""
|
||
fresh_row = {
|
||
"last_success_at": _ago(0.5),
|
||
"last_attempt_at": _ago(0.5),
|
||
"upd_24h": 10,
|
||
"upd_7d": 100,
|
||
"last_status": "done",
|
||
}
|
||
rows = {
|
||
s.table: dict(fresh_row) for s in admin_scrape._FRESHNESS_SOURCES
|
||
}
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
body = resp.json()
|
||
assert body["overall_status"] == "ok"
|
||
assert all(s["status"] == "ok" for s in body["sources"])
|
||
assert all(s["age_days"] is not None for s in body["sources"])
|
||
|
||
|
||
def test_freshness_critical_source_failed_drives_overall() -> None:
|
||
"""Критичный источник (objective) без успехов → overall_status=failed."""
|
||
base = {
|
||
"last_success_at": _ago(0.5),
|
||
"last_attempt_at": _ago(0.5),
|
||
"upd_24h": 1,
|
||
"upd_7d": 1,
|
||
"last_status": "done",
|
||
}
|
||
rows = {s.table: dict(base) for s in admin_scrape._FRESHNESS_SOURCES}
|
||
# objective критичный — обнулим успех.
|
||
rows["objective_scrape_runs"] = {
|
||
"last_success_at": None,
|
||
"last_attempt_at": _ago(5.0),
|
||
"upd_24h": 0,
|
||
"upd_7d": 0,
|
||
"last_status": "running",
|
||
}
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
body = resp.json()
|
||
assert body["overall_status"] == "failed"
|
||
|
||
|
||
def test_classify_freshness_boundaries() -> None:
|
||
"""Юнит на пороги классификатора: границы fresh/stale/critical, unknown."""
|
||
f = admin_scrape._classify_freshness
|
||
assert f(None, 2.0, 5.0) == "unknown"
|
||
assert f(1.9, 2.0, 5.0) == "fresh"
|
||
assert f(2.0, 2.0, 5.0) == "stale" # ровно fresh_days → уже stale
|
||
assert f(4.9, 2.0, 5.0) == "stale"
|
||
assert f(5.0, 2.0, 5.0) == "critical" # ровно stale_days → critical
|
||
assert f(100.0, 2.0, 5.0) == "critical"
|
||
|
||
|
||
def test_kn_thresholds_match_weekly_cadence() -> None:
|
||
"""kn шедулится еженедельно (Mon cron) → пороги должны покрывать полную неделю,
|
||
иначе ложный 'stale' каждую среду-воскресенье. Регрессия на калибровку."""
|
||
kn = next(s for s in admin_scrape._FRESHNESS_SOURCES if s.source == "kn")
|
||
f = admin_scrape._classify_freshness
|
||
# В течение недели после понедельничного прогона kn остаётся свежим.
|
||
assert f(3.9, kn.fresh_days, kn.stale_days) == "fresh" # пятница
|
||
assert f(6.5, kn.fresh_days, kn.stale_days) == "fresh" # воскресенье
|
||
# Пропуск одного понедельника → stale; двух → critical.
|
||
assert f(9.0, kn.fresh_days, kn.stale_days) == "stale"
|
||
assert f(15.0, kn.fresh_days, kn.stale_days) == "critical"
|