All checks were successful
CI Trade-In / changes (pull_request) Successful in 8s
CI Trade-In / frontend-checks (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI Trade-In / backend-tests (pull_request) Successful in 1m36s
CI / changes (pull_request) Successful in 8s
CI / backend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
Джоба-статус rosreestr_* остаётся done при quarter-stale данных (inserted=0 == всё импортировано), маскируя пропущенный операторский импорт нового квартала. Новый daily-монитор смотрит max(deal_date) и поднимает ERROR-алерт (→ GlitchTip), когда следующий квартал просрочен публикацией (порог = конец имеющегося квартала + 3 мес + 45д лага). Расследование (2026-07-02): импорт НЕ сломан — deal_date = лейбл начала квартала, 2026Q1 = последний опубликованный Росреестром (Q2 закончился 30.06, публикация ожидается ~август; исторические кварталы загружены разово 26.04). Монитор ловит именно пропуск ОПЕРАТОРСКОГО шага (data/sql/02_load_all_quarters.sh), когда quarter_poll задетектит Q2. - app/tasks/deals_freshness_monitor.py — чистая evaluate_deals_freshness (frozen-now testable) + DB-обёртка; overdue → ERROR-лог, не failed - scheduler.py + product_handlers.py — оба dispatch-пути (kit боевой + legacy) - data/sql/162 — seed (daily, 08:00-09:00 UTC, lag_allowance_days=45) - 23 теста: арифметика порога (год-переход, clamp), migration props, parity Refs #2212
251 lines
9.3 KiB
Python
251 lines
9.3 KiB
Python
"""Freshness-монитор данных deals по max(deal_date) — #2212.
|
||
|
||
Покрывает:
|
||
1. Чистую логику evaluate_deals_freshness (frozen now, без БД):
|
||
- нет алерта: 2026Q1 @ 2026-07-02 (порог 2026-08-15 не наступил);
|
||
- алерт: 2026Q1 @ 2026-08-20 (days_overdue=5);
|
||
- граница порога (== threshold → нет алерта; +1 день → алерт);
|
||
- кастомный lag_allowance_days.
|
||
2. check_deals_freshness с FakeDB (no-alert / alert / empty→mark_failed).
|
||
3. Свойства миграции 162 (по образцу test_deactivate_stale_listings).
|
||
4. Регистрацию в scheduler-dispatch + kit product_handlers (registry остаётся зелёным).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import inspect
|
||
import os
|
||
import re
|
||
from datetime import UTC, date, datetime
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from app.services import scheduler
|
||
from app.services.product_handlers import build_product_handlers
|
||
from app.tasks import deals_freshness_monitor as mon
|
||
|
||
_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql"
|
||
_MIGRATION_162 = _SQL_DIR / "162_seed_deals_freshness_monitor.sql"
|
||
|
||
# max(deal_date) = 2026Q1 label (проверено на проде 2026-07-02).
|
||
_Q1_2026 = date(2026, 1, 1)
|
||
|
||
|
||
# ── evaluate_deals_freshness (чистая логика, frozen now) ──────────────────────
|
||
|
||
|
||
def _now(y: int, m: int, d: int) -> datetime:
|
||
return datetime(y, m, d, tzinfo=UTC)
|
||
|
||
|
||
def test_no_alert_now_2026_07_02() -> None:
|
||
"""Сейчас: 2026Q1 @ 2026-07-02 — порог просрочки 2026-08-15 не наступил."""
|
||
v = mon.evaluate_deals_freshness(_Q1_2026, _now(2026, 7, 2))
|
||
assert v.overdue is False
|
||
assert v.days_overdue == 0
|
||
assert v.latest_q_end == date(2026, 4, 1)
|
||
assert v.overdue_threshold == date(2026, 8, 15)
|
||
|
||
|
||
def test_alert_2026_08_20() -> None:
|
||
"""К 2026-08-20 Q2 не импортирован — алерт, просрочка 5 дней (порог 2026-08-15)."""
|
||
v = mon.evaluate_deals_freshness(_Q1_2026, _now(2026, 8, 20))
|
||
assert v.overdue is True
|
||
assert v.days_overdue == 5
|
||
|
||
|
||
def test_threshold_boundary_exact_no_alert() -> None:
|
||
"""Ровно в день порога (2026-08-15) алерта ещё нет (строгое >)."""
|
||
v = mon.evaluate_deals_freshness(_Q1_2026, _now(2026, 8, 15))
|
||
assert v.overdue is False
|
||
assert v.days_overdue == 0
|
||
|
||
|
||
def test_threshold_boundary_next_day_alert() -> None:
|
||
"""Порог + 1 день (2026-08-16) — первый алерт, просрочка 1 день."""
|
||
v = mon.evaluate_deals_freshness(_Q1_2026, _now(2026, 8, 16))
|
||
assert v.overdue is True
|
||
assert v.days_overdue == 1
|
||
|
||
|
||
def test_custom_lag_allowance_shifts_threshold() -> None:
|
||
"""lag_allowance_days=0 сдвигает порог на конец следующего квартала (2026-07-01)."""
|
||
v = mon.evaluate_deals_freshness(_Q1_2026, _now(2026, 7, 2), lag_allowance_days=0)
|
||
assert v.overdue_threshold == date(2026, 7, 1)
|
||
assert v.overdue is True
|
||
assert v.days_overdue == 1
|
||
|
||
|
||
def test_add_months_quarter_starts() -> None:
|
||
assert mon._add_months(date(2026, 1, 1), 3) == date(2026, 4, 1)
|
||
assert mon._add_months(date(2026, 1, 1), 6) == date(2026, 7, 1)
|
||
assert mon._add_months(date(2026, 10, 1), 3) == date(2027, 1, 1)
|
||
|
||
|
||
def test_add_months_end_of_month_clamp() -> None:
|
||
# 31 янв + 1 мес → 28 фев (зажим на конец месяца).
|
||
assert mon._add_months(date(2026, 1, 31), 1) == date(2026, 2, 28)
|
||
|
||
|
||
# ── check_deals_freshness (FakeDB) ────────────────────────────────────────────
|
||
|
||
|
||
class _Row:
|
||
def __init__(self, latest: date | None) -> None:
|
||
self.latest = latest
|
||
|
||
|
||
class _FakeResult:
|
||
def __init__(self, latest: date | None) -> None:
|
||
self._latest = latest
|
||
|
||
def first(self) -> _Row:
|
||
return _Row(self._latest)
|
||
|
||
|
||
class _FakeDB:
|
||
def __init__(self, latest: date | None) -> None:
|
||
self._latest = latest
|
||
self.rolled_back = False
|
||
|
||
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
|
||
return _FakeResult(self._latest)
|
||
|
||
def rollback(self) -> None:
|
||
self.rolled_back = True
|
||
|
||
|
||
def _patch_runs(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
|
||
calls: dict[str, Any] = {"done": None, "failed": None, "heartbeat": 0}
|
||
monkeypatch.setattr(
|
||
mon.runs_mod,
|
||
"update_heartbeat",
|
||
lambda *a, **k: calls.__setitem__("heartbeat", calls["heartbeat"] + 1),
|
||
)
|
||
monkeypatch.setattr(
|
||
mon.runs_mod,
|
||
"mark_done",
|
||
lambda _db, run_id, counters: calls.__setitem__("done", dict(counters)),
|
||
)
|
||
monkeypatch.setattr(
|
||
mon.runs_mod,
|
||
"mark_failed",
|
||
lambda _db, run_id, err, counters: calls.__setitem__("failed", err),
|
||
)
|
||
return calls
|
||
|
||
|
||
def test_check_no_alert_marks_done(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
calls = _patch_runs(monkeypatch)
|
||
db = _FakeDB(_Q1_2026)
|
||
out = mon.check_deals_freshness(db, run_id=1, params={}, now=_now(2026, 7, 2)) # type: ignore[arg-type]
|
||
assert out == {"latest_year": 2026, "latest_quarter": 1, "days_overdue": 0, "alert": 0}
|
||
assert calls["done"] == out
|
||
assert calls["failed"] is None
|
||
|
||
|
||
def test_check_overdue_marks_done_with_alert(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
calls = _patch_runs(monkeypatch)
|
||
db = _FakeDB(_Q1_2026)
|
||
out = mon.check_deals_freshness(db, run_id=2, params={}, now=_now(2026, 8, 20)) # type: ignore[arg-type]
|
||
assert out["alert"] == 1
|
||
assert out["days_overdue"] == 5
|
||
# Монитор НЕ падает при алерте — прогон done, а не failed.
|
||
assert calls["done"] == out
|
||
assert calls["failed"] is None
|
||
|
||
|
||
def test_check_empty_deals_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
calls = _patch_runs(monkeypatch)
|
||
db = _FakeDB(None)
|
||
out = mon.check_deals_freshness(db, run_id=3, params={}, now=_now(2026, 7, 2)) # type: ignore[arg-type]
|
||
assert out["alert"] == 0
|
||
assert calls["done"] is None
|
||
assert calls["failed"] is not None
|
||
|
||
|
||
def test_check_reads_lag_from_params(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
_patch_runs(monkeypatch)
|
||
db = _FakeDB(_Q1_2026)
|
||
# lag=0 → порог 2026-07-01 → @2026-07-02 уже просрочено.
|
||
out = mon.check_deals_freshness(
|
||
db, run_id=4, params={"lag_allowance_days": 0}, now=_now(2026, 7, 2)
|
||
) # type: ignore[arg-type]
|
||
assert out["alert"] == 1
|
||
|
||
|
||
# ── Миграция 162 ──────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_migration_162_exists() -> None:
|
||
assert _MIGRATION_162.is_file(), f"missing migration: {_MIGRATION_162}"
|
||
|
||
|
||
def test_migration_162_seeds_source() -> None:
|
||
sql = _MIGRATION_162.read_text("utf-8")
|
||
assert "'deals_freshness_monitor'" in sql
|
||
|
||
|
||
def test_migration_162_enabled_true() -> None:
|
||
sql = _MIGRATION_162.read_text("utf-8")
|
||
assert "true" in sql
|
||
|
||
|
||
def test_migration_162_is_idempotent() -> None:
|
||
sql = _MIGRATION_162.read_text("utf-8")
|
||
assert "ON CONFLICT (source) DO NOTHING" in sql
|
||
|
||
|
||
def test_migration_162_is_transactional() -> None:
|
||
sql = _MIGRATION_162.read_text("utf-8")
|
||
assert "BEGIN;" in sql
|
||
assert "COMMIT;" in sql
|
||
|
||
|
||
def test_migration_162_window_8_to_9_utc() -> None:
|
||
sql = _MIGRATION_162.read_text("utf-8")
|
||
assert re.search(r"\b8\b", sql), "window_start_hour 8 missing"
|
||
assert re.search(r"\b9\b", sql), "window_end_hour 9 missing"
|
||
|
||
|
||
def test_migration_162_lag_allowance_45() -> None:
|
||
sql = _MIGRATION_162.read_text("utf-8")
|
||
assert "lag_allowance_days" in sql
|
||
assert "45" in sql
|
||
|
||
|
||
def test_migration_162_no_psycopg_trap() -> None:
|
||
sql = _MIGRATION_162.read_text("utf-8")
|
||
assert not re.search(r":\w+::", sql)
|
||
|
||
|
||
# ── Регистрация в scheduler + kit registry ────────────────────────────────────
|
||
|
||
|
||
def test_trigger_exists_and_is_async() -> None:
|
||
fn = getattr(scheduler, "trigger_deals_freshness_monitor_run", None)
|
||
assert fn is not None, "trigger_deals_freshness_monitor_run missing from scheduler"
|
||
assert inspect.iscoroutinefunction(fn)
|
||
|
||
|
||
def test_dispatch_branch_present() -> None:
|
||
loop_src = inspect.getsource(scheduler.scheduler_loop)
|
||
assert 'source == "deals_freshness_monitor"' in loop_src
|
||
assert "trigger_deals_freshness_monitor_run(db, sch)" in loop_src
|
||
|
||
|
||
def test_trigger_runs_sync_task_in_executor() -> None:
|
||
trig_src = inspect.getsource(scheduler.trigger_deals_freshness_monitor_run)
|
||
assert "run_in_executor(" in trig_src
|
||
assert "check_deals_freshness" in trig_src
|
||
assert "_claim_run(db, schedule_row)" in trig_src
|
||
|
||
|
||
def test_kit_product_handler_registered() -> None:
|
||
# ctx не нужен для сборки dict ключей — build_product_handlers его не замыкает.
|
||
handlers = build_product_handlers(ctx=None) # type: ignore[arg-type]
|
||
assert "deals_freshness_monitor" in handlers
|