Light1YT
bd1adc1f59
The data-freshness monitor classified by run RECENCY only, so the domrf_kn FLATS loader running status=done but extracting 0 flats for ~5 weeks went undetected — and the kn source watched objects_count (healthy ~1548), not flats_count (the broken =0 metric). Add an opt-in zero-output check: an otherwise-fresh run-ledger source (recent success, would-be fresh by age) that produced 0 work-rows in the 7d window is downgraded to status="failed" (so scrape_freshness_check alerts), with an additive "reason". Guards: alert_on_zero_output flag, run-ledger only (timestamp_col is None), status=="ok" (age-stale/failed already covered), and upd_7d==0 (SUM of the source's own work_col over done-runs). Registry: new kn_flats source (kn_scrape_runs, work_col=flats_count, critical, flag on) — watches the column that was broken; existing kn (objects_count) unchanged. Flag also enabled on objective (rows_lots, critical). nspd/nspd_geo/ cadastre left unflagged (legitimate-0 / data-table). JSON additive only (new nullable "reason" key; endpoint is dict[str,Any], no frontend consumer / no codegen needed). 4 new tests (downgrade, no-false- positive, age-precedence, registry). code-reviewer APPROVE. Would have caught #1945 within ~8-14d instead of 5 weeks.
450 lines
18 KiB
Python
450 lines
18 KiB
Python
"""Тесты для GET /api/v1/admin/scrape/freshness (#73 B5-3).
|
||
|
||
Эндпоинт читает 6 источников по 5 run-/data-таблицам (kn / kn_flats / objective /
|
||
nspd / nspd_geo / cadastre — kn и kn_flats делят kn_scrape_runs, разные work_col)
|
||
и для каждого считает свежесть. Здесь DB замокан: на каждый per-source SELECT
|
||
возвращаем заранее заданную строку, чтобы проверить классификацию
|
||
fresh/stale/critical, маппинг status (ok/stale/failed), zero-output downgrade
|
||
(#1945) и агрегат overall_status по критичным источникам.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
from datetime import UTC, datetime, timedelta
|
||
from typing import Any
|
||
from unittest.mock import MagicMock
|
||
|
||
from fastapi.testclient import TestClient
|
||
|
||
from app.api.v1 import admin_scrape
|
||
from app.core.db import get_db
|
||
from app.main import app
|
||
|
||
ENDPOINT = "/api/v1/admin/scrape/freshness"
|
||
|
||
|
||
def _ago(days: float) -> datetime:
|
||
return datetime.now(UTC) - timedelta(days=days)
|
||
|
||
|
||
def _make_db(per_source: dict[str, dict[str, Any]]) -> MagicMock:
|
||
"""Db-mock: каждый execute()'s SELECT относится к одной run-таблице; матчим по
|
||
имени таблицы в SQL и возвращаем соответствующую заранее заданную строку.
|
||
|
||
per_source: ключ — имя run-таблицы, значение — mapping-строка (как .first()).
|
||
"""
|
||
db = MagicMock()
|
||
|
||
def execute(stmt: Any, params: Any = None) -> Any:
|
||
sql = str(stmt)
|
||
matched: dict[str, Any] | None = None
|
||
# Матчим по FROM <table> — самый длинный матч (nspd_geo_jobs vs nspd_*).
|
||
for table in sorted(per_source, key=len, reverse=True):
|
||
if f"FROM {table}" in sql:
|
||
matched = per_source[table]
|
||
break
|
||
result = MagicMock()
|
||
result.mappings.return_value.first.return_value = matched
|
||
return result
|
||
|
||
db.execute = execute
|
||
return db
|
||
|
||
|
||
def _override_db(db: MagicMock) -> None:
|
||
def _factory():
|
||
yield db
|
||
|
||
app.dependency_overrides[get_db] = _factory
|
||
|
||
|
||
def test_freshness_mixed_statuses() -> None:
|
||
"""kn свежий (ok), objective stale, nspd (data-table) свежий (ok), nspd_geo critical,
|
||
cadastre fresh. overall_status = stale (по критичному objective→stale)."""
|
||
rows = {
|
||
"kn_scrape_runs": {
|
||
"last_success_at": _ago(1.0), # < fresh_days=2 → fresh → ok
|
||
"last_attempt_at": _ago(1.0),
|
||
"upd_24h": 1500,
|
||
"upd_7d": 1500,
|
||
"last_status": "done",
|
||
},
|
||
"objective_scrape_runs": {
|
||
"last_success_at": _ago(10.0), # fresh<7, stale<14 → stale
|
||
"last_attempt_at": _ago(10.0),
|
||
"upd_24h": 0,
|
||
"upd_7d": 325000,
|
||
"last_status": "done",
|
||
},
|
||
# nspd теперь data-table режим (nspd_quarter_dumps.fetched_at_utc):
|
||
# last_success_at = MAX(timestamp_col), last_status = NULL (нет run-ledger).
|
||
"nspd_quarter_dumps": {
|
||
"last_success_at": _ago(4.0), # < fresh_days=14 → fresh → ok
|
||
"last_attempt_at": _ago(4.0),
|
||
"upd_24h": 5,
|
||
"upd_7d": 56,
|
||
"last_status": None,
|
||
},
|
||
"nspd_geo_jobs": {
|
||
"last_success_at": _ago(40.0), # >= stale_days=30 → critical → failed
|
||
"last_attempt_at": _ago(40.0),
|
||
"upd_24h": 0,
|
||
"upd_7d": 0,
|
||
"last_status": "done",
|
||
},
|
||
"cadastre_jobs": {
|
||
"last_success_at": _ago(3.0), # < fresh_days=14 → fresh → ok
|
||
"last_attempt_at": _ago(3.0),
|
||
"upd_24h": 5,
|
||
"upd_7d": 5,
|
||
"last_status": "done",
|
||
},
|
||
}
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
body = resp.json()
|
||
assert "generated_at" in body
|
||
# kn / kn_flats / objective / nspd / nspd_geo / cadastre.
|
||
assert len(body["sources"]) == len(admin_scrape._FRESHNESS_SOURCES)
|
||
by_src = {s["source"]: s for s in body["sources"]}
|
||
|
||
assert by_src["kn"]["status"] == "ok"
|
||
assert by_src["kn"]["freshness"] == "fresh"
|
||
assert by_src["kn"]["objects_updated_7d"] == 1500
|
||
assert by_src["kn"]["critical"] is True
|
||
|
||
assert by_src["objective"]["status"] == "stale"
|
||
assert by_src["objective"]["freshness"] == "stale"
|
||
|
||
# nspd — data-table режим: свежий dump → ok, last_status=NULL не ломает маппинг.
|
||
assert by_src["nspd"]["status"] == "ok"
|
||
assert by_src["nspd"]["freshness"] == "fresh"
|
||
assert by_src["nspd"]["last_status"] is None
|
||
assert by_src["nspd"]["objects_updated_7d"] == 56
|
||
|
||
assert by_src["nspd_geo"]["status"] == "failed"
|
||
assert by_src["nspd_geo"]["freshness"] == "critical"
|
||
|
||
assert by_src["cadastre"]["status"] == "ok"
|
||
|
||
# overall_status — худший по критичным (kn=ok, objective=stale) → stale.
|
||
# nspd_geo критичным НЕ помечен, поэтому его failed в overall не попадает.
|
||
assert by_src["nspd_geo"]["critical"] is False
|
||
assert body["overall_status"] == "stale"
|
||
|
||
|
||
def _healthy_run_rows_except_nspd(nspd_row: dict[str, Any]) -> dict[str, dict[str, Any]]:
|
||
"""Все run-ledger источники свежие, nspd — отдельной data-table строкой.
|
||
|
||
Используется для изоляции поведения nspd-источника в data-table режиме, не
|
||
затрагивая остальные источники.
|
||
"""
|
||
fresh_run = {
|
||
"last_success_at": _ago(0.5),
|
||
"last_attempt_at": _ago(0.5),
|
||
"upd_24h": 10,
|
||
"upd_7d": 100,
|
||
"last_status": "done",
|
||
}
|
||
rows: dict[str, dict[str, Any]] = {}
|
||
for s in admin_scrape._FRESHNESS_SOURCES:
|
||
rows[s.table] = nspd_row if s.source == "nspd" else dict(fresh_run)
|
||
return rows
|
||
|
||
|
||
def test_nspd_uses_timestamp_col_mode_fresh() -> None:
|
||
"""nspd теперь читает data-таблицу nspd_quarter_dumps по timestamp_col=fetched_at_utc.
|
||
|
||
Свежий dump (~4d < fresh_days=14) + last_status=NULL → freshness=fresh, status=ok
|
||
(НЕ failed). Регрессия на repoint defunct nspd_scrape_runs → живой harvest-dump.
|
||
"""
|
||
nspd = next(s for s in admin_scrape._FRESHNESS_SOURCES if s.source == "nspd")
|
||
# Подтверждаем что источник в data-table режиме и указывает на живую таблицу.
|
||
assert nspd.timestamp_col == "fetched_at_utc"
|
||
assert nspd.table == "nspd_quarter_dumps"
|
||
assert nspd.critical is False # остаётся некритичным
|
||
|
||
rows = _healthy_run_rows_except_nspd(
|
||
{
|
||
"last_success_at": _ago(4.0), # < fresh_days=14 → fresh
|
||
"last_attempt_at": _ago(4.0),
|
||
"upd_24h": 5,
|
||
"upd_7d": 56,
|
||
"last_status": None, # data-table → NULL, маппинг не должен падать
|
||
}
|
||
)
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
by_src = {s["source"]: s for s in resp.json()["sources"]}
|
||
assert by_src["nspd"]["freshness"] == "fresh"
|
||
assert by_src["nspd"]["status"] == "ok"
|
||
assert by_src["nspd"]["last_status"] is None
|
||
assert by_src["nspd"]["objects_updated_7d"] == 56
|
||
assert by_src["nspd"]["age_days"] is not None and by_src["nspd"]["age_days"] < 14.0
|
||
|
||
|
||
def test_nspd_timestamp_col_stale_dump_is_critical() -> None:
|
||
"""Старый dump (40d >= stale_days=30) в data-table режиме → critical → status=failed."""
|
||
rows = _healthy_run_rows_except_nspd(
|
||
{
|
||
"last_success_at": _ago(40.0), # >= stale_days=30 → critical
|
||
"last_attempt_at": _ago(40.0),
|
||
"upd_24h": 0,
|
||
"upd_7d": 0,
|
||
"last_status": None,
|
||
}
|
||
)
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
by_src = {s["source"]: s for s in resp.json()["sources"]}
|
||
assert by_src["nspd"]["freshness"] == "critical"
|
||
assert by_src["nspd"]["status"] == "failed"
|
||
|
||
|
||
def test_freshness_all_healthy() -> None:
|
||
"""Все источники свежие → overall_status=ok."""
|
||
fresh_row = {
|
||
"last_success_at": _ago(0.5),
|
||
"last_attempt_at": _ago(0.5),
|
||
"upd_24h": 10,
|
||
"upd_7d": 100,
|
||
"last_status": "done",
|
||
}
|
||
rows = {
|
||
s.table: dict(fresh_row) for s in admin_scrape._FRESHNESS_SOURCES
|
||
}
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
body = resp.json()
|
||
assert body["overall_status"] == "ok"
|
||
assert all(s["status"] == "ok" for s in body["sources"])
|
||
assert all(s["age_days"] is not None for s in body["sources"])
|
||
|
||
|
||
def test_freshness_critical_source_failed_drives_overall() -> None:
|
||
"""Критичный источник (objective) без успехов → overall_status=failed."""
|
||
base = {
|
||
"last_success_at": _ago(0.5),
|
||
"last_attempt_at": _ago(0.5),
|
||
"upd_24h": 1,
|
||
"upd_7d": 1,
|
||
"last_status": "done",
|
||
}
|
||
rows = {s.table: dict(base) for s in admin_scrape._FRESHNESS_SOURCES}
|
||
# objective критичный — обнулим успех.
|
||
rows["objective_scrape_runs"] = {
|
||
"last_success_at": None,
|
||
"last_attempt_at": _ago(5.0),
|
||
"upd_24h": 0,
|
||
"upd_7d": 0,
|
||
"last_status": "running",
|
||
}
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
body = resp.json()
|
||
assert body["overall_status"] == "failed"
|
||
|
||
|
||
def test_classify_freshness_boundaries() -> None:
|
||
"""Юнит на пороги классификатора: границы fresh/stale/critical, unknown."""
|
||
f = admin_scrape._classify_freshness
|
||
assert f(None, 2.0, 5.0) == "unknown"
|
||
assert f(1.9, 2.0, 5.0) == "fresh"
|
||
assert f(2.0, 2.0, 5.0) == "stale" # ровно fresh_days → уже stale
|
||
assert f(4.9, 2.0, 5.0) == "stale"
|
||
assert f(5.0, 2.0, 5.0) == "critical" # ровно stale_days → critical
|
||
assert f(100.0, 2.0, 5.0) == "critical"
|
||
|
||
|
||
def test_zero_output_downgrades_otherwise_fresh_source() -> None:
|
||
"""issue #1945: kn_flats прогнался успешно И недавно (был бы fresh→ok по возрасту),
|
||
но flats_count=0 за 7d done-прогонов → DOWNGRADE до failed (alertable).
|
||
|
||
Та же run-таблица kn_scrape_runs у kn (alert_on_zero_output=False) и kn_flats
|
||
(True). Возвращаем строку с upd_7d=0 → kn остаётся ok (нет флага = нет ложного
|
||
срабатывания), kn_flats падает в failed (zero-output) с пояснением в reason.
|
||
"""
|
||
fresh_zero = {
|
||
"last_success_at": _ago(1.0), # < fresh_days=8 → был бы fresh
|
||
"last_attempt_at": _ago(1.0),
|
||
"upd_24h": 0,
|
||
"upd_7d": 0, # 0 work-строк за окно done-прогонов → zero-output
|
||
"last_status": "done",
|
||
}
|
||
# Остальные источники свежие+ненулевые, чтобы изолировать поведение kn/kn_flats.
|
||
nonzero_fresh = {
|
||
"last_success_at": _ago(0.5),
|
||
"last_attempt_at": _ago(0.5),
|
||
"upd_24h": 10,
|
||
"upd_7d": 100,
|
||
"last_status": "done",
|
||
}
|
||
rows: dict[str, dict[str, Any]] = {}
|
||
for s in admin_scrape._FRESHNESS_SOURCES:
|
||
if s.timestamp_col is not None:
|
||
rows[s.table] = {**nonzero_fresh, "last_status": None}
|
||
elif s.table == "kn_scrape_runs":
|
||
rows[s.table] = dict(fresh_zero)
|
||
else:
|
||
rows[s.table] = dict(nonzero_fresh)
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
body = resp.json()
|
||
by_src = {s["source"]: s for s in body["sources"]}
|
||
|
||
# kn_flats: otherwise-fresh, но flats_count=0 → DOWNGRADE до failed (alertable).
|
||
assert by_src["kn_flats"]["freshness"] == "fresh" # по возрасту был бы fresh
|
||
assert by_src["kn_flats"]["status"] == "failed" # zero-output downgrade
|
||
assert by_src["kn_flats"]["objects_updated_7d"] == 0
|
||
assert by_src["kn_flats"]["reason"] is not None
|
||
assert "flats_count" in by_src["kn_flats"]["reason"]
|
||
|
||
# kn — та же таблица, upd_7d=0, но БЕЗ флага → остаётся ok (no false positive).
|
||
assert by_src["kn"]["status"] == "ok"
|
||
assert by_src["kn"]["reason"] is None
|
||
|
||
# critical kn_flats упал → overall_status=failed.
|
||
assert by_src["kn_flats"]["critical"] is True
|
||
assert body["overall_status"] == "failed"
|
||
|
||
|
||
def test_zero_output_no_false_positive_when_nonzero() -> None:
|
||
"""alert_on_zero_output=True источник с НЕнулевым выходом → остаётся ok (нет даунгрейда).
|
||
|
||
Регрессия: zero-output даунгрейд НЕ должен бить по здоровому источнику, который
|
||
прогнался и реально извлёк строки.
|
||
"""
|
||
nonzero_fresh = {
|
||
"last_success_at": _ago(1.0),
|
||
"last_attempt_at": _ago(1.0),
|
||
"upd_24h": 12,
|
||
"upd_7d": 1234, # ненулевой выход
|
||
"last_status": "done",
|
||
}
|
||
rows: dict[str, dict[str, Any]] = {}
|
||
for s in admin_scrape._FRESHNESS_SOURCES:
|
||
rows[s.table] = (
|
||
{**nonzero_fresh, "last_status": None}
|
||
if s.timestamp_col is not None
|
||
else dict(nonzero_fresh)
|
||
)
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
by_src = {s["source"]: s for s in resp.json()["sources"]}
|
||
# Все alert_on_zero_output-источники с ненулевым выходом → ok, reason=None.
|
||
for s in admin_scrape._FRESHNESS_SOURCES:
|
||
if s.alert_on_zero_output:
|
||
assert by_src[s.source]["status"] == "ok", s.source
|
||
assert by_src[s.source]["reason"] is None, s.source
|
||
|
||
|
||
def test_zero_output_not_applied_when_already_stale() -> None:
|
||
"""Если источник уже stale/failed по возрасту — zero-output НЕ передоунгрейдит
|
||
(age уже покрывает), и не двойнит. objective: старый успех (10d → stale) + upd_7d=0
|
||
→ остаётся stale (НЕ перепрыгивает в failed через zero-output)."""
|
||
base_fresh = {
|
||
"last_success_at": _ago(0.5),
|
||
"last_attempt_at": _ago(0.5),
|
||
"upd_24h": 5,
|
||
"upd_7d": 50,
|
||
"last_status": "done",
|
||
}
|
||
rows: dict[str, dict[str, Any]] = {}
|
||
for s in admin_scrape._FRESHNESS_SOURCES:
|
||
rows[s.table] = (
|
||
{**base_fresh, "last_status": None}
|
||
if s.timestamp_col is not None
|
||
else dict(base_fresh)
|
||
)
|
||
# objective: stale по возрасту (10d: fresh<7, stale<14) + 0 выход.
|
||
rows["objective_scrape_runs"] = {
|
||
"last_success_at": _ago(10.0),
|
||
"last_attempt_at": _ago(10.0),
|
||
"upd_24h": 0,
|
||
"upd_7d": 0,
|
||
"last_status": "done",
|
||
}
|
||
_override_db(_make_db(rows))
|
||
try:
|
||
resp = TestClient(app).get(ENDPOINT)
|
||
finally:
|
||
app.dependency_overrides.clear()
|
||
|
||
assert resp.status_code == 200, resp.text
|
||
by_src = {s["source"]: s for s in resp.json()["sources"]}
|
||
# zero-output применяется ТОЛЬКО к otherwise-ok; stale остаётся stale, reason=None.
|
||
assert by_src["objective"]["status"] == "stale"
|
||
assert by_src["objective"]["reason"] is None
|
||
|
||
|
||
def test_kn_flats_registry_config() -> None:
|
||
"""kn_flats: та же run-таблица что у kn, но work_col=flats_count + alert_on_zero_output,
|
||
критичный, недельные пороги. kn (objects_count) остаётся без флага (unchanged)."""
|
||
kn_flats = next(s for s in admin_scrape._FRESHNESS_SOURCES if s.source == "kn_flats")
|
||
assert kn_flats.table == "kn_scrape_runs"
|
||
assert kn_flats.work_col == "flats_count"
|
||
assert kn_flats.timestamp_col is None # run-ledger режим
|
||
assert kn_flats.alert_on_zero_output is True
|
||
assert kn_flats.critical is True
|
||
assert kn_flats.fresh_days == 8.0
|
||
assert kn_flats.stale_days == 14.0
|
||
|
||
kn = next(s for s in admin_scrape._FRESHNESS_SOURCES if s.source == "kn")
|
||
assert kn.work_col == "objects_count"
|
||
assert kn.alert_on_zero_output is False # существующий источник unchanged
|
||
|
||
objective = next(s for s in admin_scrape._FRESHNESS_SOURCES if s.source == "objective")
|
||
assert objective.alert_on_zero_output is True
|
||
|
||
# Legitimate-0 источники НЕ должны иметь флаг (избегаем ложных срабатываний).
|
||
for name in ("nspd", "nspd_geo", "cadastre"):
|
||
src = next(s for s in admin_scrape._FRESHNESS_SOURCES if s.source == name)
|
||
assert src.alert_on_zero_output is False, name
|
||
|
||
|
||
def test_kn_thresholds_match_weekly_cadence() -> None:
|
||
"""kn шедулится еженедельно (Mon cron) → пороги должны покрывать полную неделю,
|
||
иначе ложный 'stale' каждую среду-воскресенье. Регрессия на калибровку."""
|
||
kn = next(s for s in admin_scrape._FRESHNESS_SOURCES if s.source == "kn")
|
||
f = admin_scrape._classify_freshness
|
||
# В течение недели после понедельничного прогона kn остаётся свежим.
|
||
assert f(3.9, kn.fresh_days, kn.stale_days) == "fresh" # пятница
|
||
assert f(6.5, kn.fresh_days, kn.stale_days) == "fresh" # воскресенье
|
||
# Пропуск одного понедельника → stale; двух → critical.
|
||
assert f(9.0, kn.fresh_days, kn.stale_days) == "stale"
|
||
assert f(15.0, kn.fresh_days, kn.stale_days) == "critical"
|