gendesign/tradein-mvp/backend/tests/test_deals_freshness_monitor.py
bot-backend 30d91be796
All checks were successful
CI Trade-In / changes (pull_request) Successful in 8s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 1m36s
CI / changes (pull_request) Successful in 8s
CI / backend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
feat(tradein/monitor): алерт на staleness данных deals по max(deal_date) (#2212)
Джоба-статус rosreestr_* остаётся done при quarter-stale данных (inserted=0
== всё импортировано), маскируя пропущенный операторский импорт нового
квартала. Новый daily-монитор смотрит max(deal_date) и поднимает ERROR-алерт
(→ GlitchTip), когда следующий квартал просрочен публикацией
(порог = конец имеющегося квартала + 3 мес + 45д лага).

Расследование (2026-07-02): импорт НЕ сломан — deal_date = лейбл начала
квартала, 2026Q1 = последний опубликованный Росреестром (Q2 закончился
30.06, публикация ожидается ~август; исторические кварталы загружены
разово 26.04). Монитор ловит именно пропуск ОПЕРАТОРСКОГО шага
(data/sql/02_load_all_quarters.sh), когда quarter_poll задетектит Q2.

- app/tasks/deals_freshness_monitor.py — чистая evaluate_deals_freshness
  (frozen-now testable) + DB-обёртка; overdue → ERROR-лог, не failed
- scheduler.py + product_handlers.py — оба dispatch-пути (kit боевой + legacy)
- data/sql/162 — seed (daily, 08:00-09:00 UTC, lag_allowance_days=45)
- 23 теста: арифметика порога (год-переход, clamp), migration props, parity

Refs #2212
2026-07-02 23:31:36 +03:00

251 lines
9.3 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Freshness-монитор данных deals по max(deal_date) — #2212.
Покрывает:
1. Чистую логику evaluate_deals_freshness (frozen now, без БД):
- нет алерта: 2026Q1 @ 2026-07-02 (порог 2026-08-15 не наступил);
- алерт: 2026Q1 @ 2026-08-20 (days_overdue=5);
- граница порога (== threshold → нет алерта; +1 день → алерт);
- кастомный lag_allowance_days.
2. check_deals_freshness с FakeDB (no-alert / alert / empty→mark_failed).
3. Свойства миграции 162 (по образцу test_deactivate_stale_listings).
4. Регистрацию в scheduler-dispatch + kit product_handlers (registry остаётся зелёным).
"""
from __future__ import annotations
import inspect
import os
import re
from datetime import UTC, date, datetime
from pathlib import Path
from typing import Any
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
from app.services import scheduler
from app.services.product_handlers import build_product_handlers
from app.tasks import deals_freshness_monitor as mon
_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql"
_MIGRATION_162 = _SQL_DIR / "162_seed_deals_freshness_monitor.sql"
# max(deal_date) = 2026Q1 label (проверено на проде 2026-07-02).
_Q1_2026 = date(2026, 1, 1)
# ── evaluate_deals_freshness (чистая логика, frozen now) ──────────────────────
def _now(y: int, m: int, d: int) -> datetime:
return datetime(y, m, d, tzinfo=UTC)
def test_no_alert_now_2026_07_02() -> None:
"""Сейчас: 2026Q1 @ 2026-07-02 — порог просрочки 2026-08-15 не наступил."""
v = mon.evaluate_deals_freshness(_Q1_2026, _now(2026, 7, 2))
assert v.overdue is False
assert v.days_overdue == 0
assert v.latest_q_end == date(2026, 4, 1)
assert v.overdue_threshold == date(2026, 8, 15)
def test_alert_2026_08_20() -> None:
"""К 2026-08-20 Q2 не импортирован — алерт, просрочка 5 дней (порог 2026-08-15)."""
v = mon.evaluate_deals_freshness(_Q1_2026, _now(2026, 8, 20))
assert v.overdue is True
assert v.days_overdue == 5
def test_threshold_boundary_exact_no_alert() -> None:
"""Ровно в день порога (2026-08-15) алерта ещё нет (строгое >)."""
v = mon.evaluate_deals_freshness(_Q1_2026, _now(2026, 8, 15))
assert v.overdue is False
assert v.days_overdue == 0
def test_threshold_boundary_next_day_alert() -> None:
"""Порог + 1 день (2026-08-16) — первый алерт, просрочка 1 день."""
v = mon.evaluate_deals_freshness(_Q1_2026, _now(2026, 8, 16))
assert v.overdue is True
assert v.days_overdue == 1
def test_custom_lag_allowance_shifts_threshold() -> None:
"""lag_allowance_days=0 сдвигает порог на конец следующего квартала (2026-07-01)."""
v = mon.evaluate_deals_freshness(_Q1_2026, _now(2026, 7, 2), lag_allowance_days=0)
assert v.overdue_threshold == date(2026, 7, 1)
assert v.overdue is True
assert v.days_overdue == 1
def test_add_months_quarter_starts() -> None:
assert mon._add_months(date(2026, 1, 1), 3) == date(2026, 4, 1)
assert mon._add_months(date(2026, 1, 1), 6) == date(2026, 7, 1)
assert mon._add_months(date(2026, 10, 1), 3) == date(2027, 1, 1)
def test_add_months_end_of_month_clamp() -> None:
# 31 янв + 1 мес → 28 фев (зажим на конец месяца).
assert mon._add_months(date(2026, 1, 31), 1) == date(2026, 2, 28)
# ── check_deals_freshness (FakeDB) ────────────────────────────────────────────
class _Row:
def __init__(self, latest: date | None) -> None:
self.latest = latest
class _FakeResult:
def __init__(self, latest: date | None) -> None:
self._latest = latest
def first(self) -> _Row:
return _Row(self._latest)
class _FakeDB:
def __init__(self, latest: date | None) -> None:
self._latest = latest
self.rolled_back = False
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
return _FakeResult(self._latest)
def rollback(self) -> None:
self.rolled_back = True
def _patch_runs(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
calls: dict[str, Any] = {"done": None, "failed": None, "heartbeat": 0}
monkeypatch.setattr(
mon.runs_mod,
"update_heartbeat",
lambda *a, **k: calls.__setitem__("heartbeat", calls["heartbeat"] + 1),
)
monkeypatch.setattr(
mon.runs_mod,
"mark_done",
lambda _db, run_id, counters: calls.__setitem__("done", dict(counters)),
)
monkeypatch.setattr(
mon.runs_mod,
"mark_failed",
lambda _db, run_id, err, counters: calls.__setitem__("failed", err),
)
return calls
def test_check_no_alert_marks_done(monkeypatch: pytest.MonkeyPatch) -> None:
calls = _patch_runs(monkeypatch)
db = _FakeDB(_Q1_2026)
out = mon.check_deals_freshness(db, run_id=1, params={}, now=_now(2026, 7, 2)) # type: ignore[arg-type]
assert out == {"latest_year": 2026, "latest_quarter": 1, "days_overdue": 0, "alert": 0}
assert calls["done"] == out
assert calls["failed"] is None
def test_check_overdue_marks_done_with_alert(monkeypatch: pytest.MonkeyPatch) -> None:
calls = _patch_runs(monkeypatch)
db = _FakeDB(_Q1_2026)
out = mon.check_deals_freshness(db, run_id=2, params={}, now=_now(2026, 8, 20)) # type: ignore[arg-type]
assert out["alert"] == 1
assert out["days_overdue"] == 5
# Монитор НЕ падает при алерте — прогон done, а не failed.
assert calls["done"] == out
assert calls["failed"] is None
def test_check_empty_deals_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None:
calls = _patch_runs(monkeypatch)
db = _FakeDB(None)
out = mon.check_deals_freshness(db, run_id=3, params={}, now=_now(2026, 7, 2)) # type: ignore[arg-type]
assert out["alert"] == 0
assert calls["done"] is None
assert calls["failed"] is not None
def test_check_reads_lag_from_params(monkeypatch: pytest.MonkeyPatch) -> None:
_patch_runs(monkeypatch)
db = _FakeDB(_Q1_2026)
# lag=0 → порог 2026-07-01 → @2026-07-02 уже просрочено.
out = mon.check_deals_freshness(
db, run_id=4, params={"lag_allowance_days": 0}, now=_now(2026, 7, 2)
) # type: ignore[arg-type]
assert out["alert"] == 1
# ── Миграция 162 ──────────────────────────────────────────────────────────────
def test_migration_162_exists() -> None:
assert _MIGRATION_162.is_file(), f"missing migration: {_MIGRATION_162}"
def test_migration_162_seeds_source() -> None:
sql = _MIGRATION_162.read_text("utf-8")
assert "'deals_freshness_monitor'" in sql
def test_migration_162_enabled_true() -> None:
sql = _MIGRATION_162.read_text("utf-8")
assert "true" in sql
def test_migration_162_is_idempotent() -> None:
sql = _MIGRATION_162.read_text("utf-8")
assert "ON CONFLICT (source) DO NOTHING" in sql
def test_migration_162_is_transactional() -> None:
sql = _MIGRATION_162.read_text("utf-8")
assert "BEGIN;" in sql
assert "COMMIT;" in sql
def test_migration_162_window_8_to_9_utc() -> None:
sql = _MIGRATION_162.read_text("utf-8")
assert re.search(r"\b8\b", sql), "window_start_hour 8 missing"
assert re.search(r"\b9\b", sql), "window_end_hour 9 missing"
def test_migration_162_lag_allowance_45() -> None:
sql = _MIGRATION_162.read_text("utf-8")
assert "lag_allowance_days" in sql
assert "45" in sql
def test_migration_162_no_psycopg_trap() -> None:
sql = _MIGRATION_162.read_text("utf-8")
assert not re.search(r":\w+::", sql)
# ── Регистрация в scheduler + kit registry ────────────────────────────────────
def test_trigger_exists_and_is_async() -> None:
fn = getattr(scheduler, "trigger_deals_freshness_monitor_run", None)
assert fn is not None, "trigger_deals_freshness_monitor_run missing from scheduler"
assert inspect.iscoroutinefunction(fn)
def test_dispatch_branch_present() -> None:
loop_src = inspect.getsource(scheduler.scheduler_loop)
assert 'source == "deals_freshness_monitor"' in loop_src
assert "trigger_deals_freshness_monitor_run(db, sch)" in loop_src
def test_trigger_runs_sync_task_in_executor() -> None:
trig_src = inspect.getsource(scheduler.trigger_deals_freshness_monitor_run)
assert "run_in_executor(" in trig_src
assert "check_deals_freshness" in trig_src
assert "_claim_run(db, schedule_row)" in trig_src
def test_kit_product_handler_registered() -> None:
# ctx не нужен для сборки dict ключей — build_product_handlers его не замыкает.
handlers = build_product_handlers(ctx=None) # type: ignore[arg-type]
assert "deals_freshness_monitor" in handlers