bot-backend
aaf2bacfa3
Ops automation — today the ДКП ingest is manual (JOBS hardcoded to 2026Q1); Q2'26 lands ~Aug 2026. Adds a monthly scheduler job that detects a new Rosreestr quarter and logs an actionable ingest alert. Detect+alert only — does NOT download the multi-GB zip or shell out from the tick. - rosreestr_poll.py: latest_loaded_quarter (MAX deal_date) + next-quarter calc + HEAD-probe dataset availability; defensive (network err → False, never raises) - tasks/rosreestr_quarter_poll.py: task wrapper (mirrors sber_index_pull) - scheduler.py: trigger_rosreestr_quarter_poll_run + dispatch branch - migration 096: scrape_schedules monthly seed, ON CONFLICT DO NOTHING Refs #888, #727, #724
102 lines
3 KiB
Python
102 lines
3 KiB
Python
"""Scheduled task: monthly poll of Rosreestr open-data portal for a new quarter (#888).
|
|
|
|
Delegates detect+alert logic to app.services.rosreestr_poll.poll_rosreestr_new_quarter().
|
|
Wired into the in-app scheduler as source='rosreestr_quarter_poll'. Triggered monthly via
|
|
the scrape_schedules seed (migration 096).
|
|
|
|
SCOPE: detect availability + log actionable alert ONLY.
|
|
Does NOT download the multi-GB ZIP or shell out to the loaders.
|
|
When available=True, the operator runs manually:
|
|
data/sql/02_load_all_quarters.sh
|
|
tradein-mvp/deploy/import-rosreestr.sh
|
|
|
|
Mirrors tasks/sber_index_pull.py: thin async wrapper around the service,
|
|
heartbeat → delegate → mark_done/mark_failed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.services import scrape_runs as runs_mod
|
|
from app.services.rosreestr_poll import poll_rosreestr_new_quarter
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
__all__ = [
|
|
"RosreestrQuarterPollResult",
|
|
"run_rosreestr_quarter_poll",
|
|
]
|
|
|
|
|
|
@dataclass
|
|
class RosreestrQuarterPollResult:
|
|
"""Result from a scheduled Rosreestr quarter-poll run."""
|
|
|
|
available: bool = False
|
|
year: int = 0
|
|
quarter: int = 0
|
|
duration_sec: float = field(default=0.0)
|
|
|
|
|
|
async def run_rosreestr_quarter_poll(
|
|
db: Session,
|
|
*,
|
|
run_id: int,
|
|
params: dict, # type: ignore[type-arg]
|
|
) -> RosreestrQuarterPollResult:
|
|
"""Execute Rosreestr quarter poll with run lifecycle management.
|
|
|
|
Wraps poll_rosreestr_new_quarter(), emitting heartbeat before the check
|
|
and marking the scrape_run as done/failed on completion.
|
|
|
|
Params (from default_params jsonb in scrape_schedules):
|
|
interval_days: int — informational only (scheduler uses next_run_at arithmetic).
|
|
"""
|
|
counters: dict[str, int] = {
|
|
"available": 0,
|
|
"year": 0,
|
|
"quarter": 0,
|
|
}
|
|
started_at = time.monotonic()
|
|
|
|
try:
|
|
runs_mod.update_heartbeat(db, run_id, counters)
|
|
|
|
result_dict = await poll_rosreestr_new_quarter(db)
|
|
|
|
available = bool(result_dict.get("available", False))
|
|
year = int(result_dict.get("year", 0))
|
|
quarter = int(result_dict.get("quarter", 0))
|
|
duration_sec = time.monotonic() - started_at
|
|
|
|
counters = {
|
|
"available": int(available),
|
|
"year": year,
|
|
"quarter": quarter,
|
|
"duration_sec": int(duration_sec),
|
|
}
|
|
runs_mod.mark_done(db, run_id, counters)
|
|
logger.info(
|
|
"rosreestr_quarter_poll run_id=%d done — available=%s Q%d %d %.1fs",
|
|
run_id,
|
|
available,
|
|
quarter,
|
|
year,
|
|
duration_sec,
|
|
)
|
|
return RosreestrQuarterPollResult(
|
|
available=available,
|
|
year=year,
|
|
quarter=quarter,
|
|
duration_sec=duration_sec,
|
|
)
|
|
|
|
except Exception as exc:
|
|
logger.exception("rosreestr_quarter_poll run_id=%d failed", run_id)
|
|
runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters)
|
|
raise
|