"""In-app scheduler — periodically triggers configured scrapes based on scrape_schedules table. Запускается в FastAPI lifespan (app.main lifespan context manager) при startup. Логика: 1. Каждые 60 sec query scrape_schedules WHERE enabled=true AND next_run_at <= NOW(). 2. Для каждого: проверить нет ли уже running run этого source — если нет, trigger. 3. После trigger — recompute next_run_at в следующих суток. 4. На old runs (orphaned status='running' > 6h без heartbeat) — пометить как 'zombie'. """ from __future__ import annotations import asyncio import logging import random from datetime import UTC, datetime, time, timedelta from typing import Any from sqlalchemy import text from sqlalchemy.orm import Session from app.core.db import SessionLocal from app.services import scrape_runs as runs_mod from app.services.scrape_pipeline import run_avito_city_sweep logger = logging.getLogger(__name__) # Loop interval — check каждую минуту SCHEDULER_TICK_SEC = 60 ZOMBIE_THRESHOLD_HOURS = 6 def compute_next_run_at( window_start_hour: int, window_end_hour: int, *, now: datetime | None = None ) -> datetime: """Pick random datetime в window [start, end) UTC, для СЛЕДУЮЩИХ суток после now. Если window_end_hour <= window_start_hour → cross-midnight window (например 22→3 → окно 22:00-23:59 ИЛИ 00:00-02:59). """ now = now or datetime.now(tz=UTC) tomorrow = (now + timedelta(days=1)).date() if window_end_hour > window_start_hour: # Обычное окно (например 2..5 → 02:00-04:59) start_seconds = window_start_hour * 3600 end_seconds = window_end_hour * 3600 rand_seconds = random.randint(start_seconds, end_seconds - 1) return datetime.combine(tomorrow, time(0, 0), tzinfo=UTC) + timedelta( seconds=rand_seconds ) else: # Cross-midnight (22..3 → 22:00-23:59 + 00:00-02:59) # Длина окна = (24-start) + end часов total_seconds = ((24 - window_start_hour) + window_end_hour) * 3600 rand_seconds = random.randint(0, total_seconds - 1) # Если rand попадает в первую часть (start..24) first_half = (24 - window_start_hour) * 3600 if rand_seconds < first_half: # Текущая дата (если ещё не наступило окно) или next day base_date = now.date() if now.hour < window_start_hour else tomorrow return datetime.combine(base_date, time(0, 0), tzinfo=UTC) + timedelta( seconds=window_start_hour * 3600 + rand_seconds ) else: # Во второй части (0..end), следующего дня offset = rand_seconds - first_half return datetime.combine(tomorrow, time(0, 0), tzinfo=UTC) + timedelta(seconds=offset) def has_running_run(db: Session, source: str) -> bool: """Есть ли активный run для source (status='running').""" row = db.execute( text( """ SELECT 1 FROM scrape_runs WHERE source = :source AND status = 'running' LIMIT 1 """ ), {"source": source}, ).fetchone() return row is not None def reap_zombies(db: Session) -> int: """Mark scrape_runs as 'zombie' если heartbeat не обновлялся > ZOMBIE_THRESHOLD_HOURS hours.""" zombie_interval = f"{ZOMBIE_THRESHOLD_HOURS} hours" result = db.execute( text( """ UPDATE scrape_runs SET status = 'zombie', finished_at = NOW() WHERE status = 'running' AND (heartbeat_at IS NULL OR heartbeat_at < NOW() - CAST(:interval AS interval)) RETURNING id """ ), {"interval": zombie_interval}, ) rows = result.fetchall() db.commit() if rows: logger.warning("scheduler: reaped %d zombie runs: %s", len(rows), [r.id for r in rows]) return len(rows) async def trigger_avito_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None: """Создать новый scrape_runs + launch run_avito_city_sweep в asyncio.create_task. Returns run_id (или None если skip — есть running run). """ if has_running_run(db, schedule_row["source"]): logger.info("scheduler: skip — already running for source=%s", schedule_row["source"]) return None params = schedule_row.get("default_params") or {} run_id = runs_mod.create_run(db, source=schedule_row["source"], params=params) # Update schedule: last_run_id, last_run_at, recompute next_run_at next_at = compute_next_run_at( schedule_row["window_start_hour"], schedule_row["window_end_hour"], ) db.execute( text( """ UPDATE scrape_schedules SET last_run_id = :run_id, last_run_at = NOW(), next_run_at = :next_at, updated_at = NOW() WHERE source = :source """ ), {"run_id": run_id, "next_at": next_at, "source": schedule_row["source"]}, ) db.commit() # Spawn asyncio task — pass NEW session (avoid sharing with this scheduler tick) async def _run() -> None: run_db = SessionLocal() try: await run_avito_city_sweep( run_db, run_id=run_id, pages_per_anchor=int(params.get("pages_per_anchor", 3)), detail_top_n=int(params.get("detail_top_n", 20)), request_delay_sec=float(params.get("request_delay_sec", 7.0)), enrich_houses=bool(params.get("enrich_houses", True)), radius_m=int(params.get("radius_m", 1500)), ) except Exception: logger.exception("scheduler: run_avito_city_sweep crashed run_id=%d", run_id) finally: run_db.close() task = asyncio.create_task(_run()) # Keep reference to avoid GC before task completes (RUF006) task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None) logger.info( "scheduler: triggered city_sweep run_id=%d next_run_at=%s", run_id, next_at.isoformat() ) return run_id def get_due_schedules(db: Session) -> list[dict[str, Any]]: """SELECT scrape_schedules WHERE enabled AND (next_run_at IS NULL OR next_run_at <= NOW()).""" rows = db.execute( text( """ SELECT id, source, enabled, window_start_hour, window_end_hour, default_params, last_run_id, last_run_at, next_run_at FROM scrape_schedules WHERE enabled = true AND (next_run_at IS NULL OR next_run_at <= NOW()) """ ), ).mappings().all() return [dict(r) for r in rows] async def scheduler_loop() -> None: """Бесконечный async loop — tick каждые SCHEDULER_TICK_SEC секунд.""" logger.info("scheduler: started (tick=%ds)", SCHEDULER_TICK_SEC) # Initial sleep 30s чтобы дать FastAPI startup завершиться await asyncio.sleep(30) while True: try: db = SessionLocal() try: # Reap zombies first reap_zombies(db) # Process due schedules due = get_due_schedules(db) for sch in due: if sch["source"] == "avito_city_sweep": await trigger_avito_city_sweep_run(db, sch) else: logger.warning("scheduler: unknown source=%s, skip", sch["source"]) finally: db.close() except Exception: logger.exception("scheduler: tick failed") await asyncio.sleep(SCHEDULER_TICK_SEC)