203 lines
7.9 KiB
Python
203 lines
7.9 KiB
Python
"""In-app scheduler — periodically triggers configured scrapes based on scrape_schedules table.
|
||
|
||
Запускается в FastAPI lifespan (app.main lifespan context manager) при startup.
|
||
Логика:
|
||
1. Каждые 60 sec query scrape_schedules WHERE enabled=true AND next_run_at <= NOW().
|
||
2. Для каждого: проверить нет ли уже running run этого source — если нет, trigger.
|
||
3. После trigger — recompute next_run_at в следующих суток.
|
||
4. На old runs (orphaned status='running' > 6h без heartbeat) — пометить как 'zombie'.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import logging
|
||
import random
|
||
from datetime import UTC, datetime, time, timedelta
|
||
from typing import Any
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.core.db import SessionLocal
|
||
from app.services import scrape_runs as runs_mod
|
||
from app.services.scrape_pipeline import run_avito_city_sweep
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Loop interval — check каждую минуту
|
||
SCHEDULER_TICK_SEC = 60
|
||
ZOMBIE_THRESHOLD_HOURS = 6
|
||
|
||
|
||
def compute_next_run_at(
|
||
window_start_hour: int, window_end_hour: int, *, now: datetime | None = None
|
||
) -> datetime:
|
||
"""Pick random datetime в window [start, end) UTC, для СЛЕДУЮЩИХ суток после now.
|
||
|
||
Если window_end_hour <= window_start_hour → cross-midnight window
|
||
(например 22→3 → окно 22:00-23:59 ИЛИ 00:00-02:59).
|
||
"""
|
||
now = now or datetime.now(tz=UTC)
|
||
tomorrow = (now + timedelta(days=1)).date()
|
||
|
||
if window_end_hour > window_start_hour:
|
||
# Обычное окно (например 2..5 → 02:00-04:59)
|
||
start_seconds = window_start_hour * 3600
|
||
end_seconds = window_end_hour * 3600
|
||
rand_seconds = random.randint(start_seconds, end_seconds - 1)
|
||
return datetime.combine(tomorrow, time(0, 0), tzinfo=UTC) + timedelta(
|
||
seconds=rand_seconds
|
||
)
|
||
else:
|
||
# Cross-midnight (22..3 → 22:00-23:59 + 00:00-02:59)
|
||
# Длина окна = (24-start) + end часов
|
||
total_seconds = ((24 - window_start_hour) + window_end_hour) * 3600
|
||
rand_seconds = random.randint(0, total_seconds - 1)
|
||
# Если rand попадает в первую часть (start..24)
|
||
first_half = (24 - window_start_hour) * 3600
|
||
if rand_seconds < first_half:
|
||
# Текущая дата (если ещё не наступило окно) или next day
|
||
base_date = now.date() if now.hour < window_start_hour else tomorrow
|
||
return datetime.combine(base_date, time(0, 0), tzinfo=UTC) + timedelta(
|
||
seconds=window_start_hour * 3600 + rand_seconds
|
||
)
|
||
else:
|
||
# Во второй части (0..end), следующего дня
|
||
offset = rand_seconds - first_half
|
||
return datetime.combine(tomorrow, time(0, 0), tzinfo=UTC) + timedelta(seconds=offset)
|
||
|
||
|
||
def has_running_run(db: Session, source: str) -> bool:
|
||
"""Есть ли активный run для source (status='running')."""
|
||
row = db.execute(
|
||
text(
|
||
"""
|
||
SELECT 1 FROM scrape_runs
|
||
WHERE source = :source AND status = 'running'
|
||
LIMIT 1
|
||
"""
|
||
),
|
||
{"source": source},
|
||
).fetchone()
|
||
return row is not None
|
||
|
||
|
||
def reap_zombies(db: Session) -> int:
|
||
"""Mark scrape_runs as 'zombie' если heartbeat не обновлялся > ZOMBIE_THRESHOLD_HOURS hours."""
|
||
zombie_interval = f"{ZOMBIE_THRESHOLD_HOURS} hours"
|
||
result = db.execute(
|
||
text(
|
||
"""
|
||
UPDATE scrape_runs
|
||
SET status = 'zombie', finished_at = NOW()
|
||
WHERE status = 'running'
|
||
AND (heartbeat_at IS NULL
|
||
OR heartbeat_at < NOW() - CAST(:interval AS interval))
|
||
RETURNING id
|
||
"""
|
||
),
|
||
{"interval": zombie_interval},
|
||
)
|
||
rows = result.fetchall()
|
||
db.commit()
|
||
if rows:
|
||
logger.warning("scheduler: reaped %d zombie runs: %s", len(rows), [r.id for r in rows])
|
||
return len(rows)
|
||
|
||
|
||
async def trigger_avito_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
|
||
"""Создать новый scrape_runs + launch run_avito_city_sweep в asyncio.create_task.
|
||
|
||
Returns run_id (или None если skip — есть running run).
|
||
"""
|
||
if has_running_run(db, schedule_row["source"]):
|
||
logger.info("scheduler: skip — already running for source=%s", schedule_row["source"])
|
||
return None
|
||
|
||
params = schedule_row.get("default_params") or {}
|
||
run_id = runs_mod.create_run(db, source=schedule_row["source"], params=params)
|
||
|
||
# Update schedule: last_run_id, last_run_at, recompute next_run_at
|
||
next_at = compute_next_run_at(
|
||
schedule_row["window_start_hour"],
|
||
schedule_row["window_end_hour"],
|
||
)
|
||
db.execute(
|
||
text(
|
||
"""
|
||
UPDATE scrape_schedules
|
||
SET last_run_id = :run_id, last_run_at = NOW(),
|
||
next_run_at = :next_at, updated_at = NOW()
|
||
WHERE source = :source
|
||
"""
|
||
),
|
||
{"run_id": run_id, "next_at": next_at, "source": schedule_row["source"]},
|
||
)
|
||
db.commit()
|
||
|
||
# Spawn asyncio task — pass NEW session (avoid sharing with this scheduler tick)
|
||
async def _run() -> None:
|
||
run_db = SessionLocal()
|
||
try:
|
||
await run_avito_city_sweep(
|
||
run_db,
|
||
run_id=run_id,
|
||
pages_per_anchor=int(params.get("pages_per_anchor", 3)),
|
||
detail_top_n=int(params.get("detail_top_n", 20)),
|
||
request_delay_sec=float(params.get("request_delay_sec", 7.0)),
|
||
enrich_houses=bool(params.get("enrich_houses", True)),
|
||
radius_m=int(params.get("radius_m", 1500)),
|
||
)
|
||
except Exception:
|
||
logger.exception("scheduler: run_avito_city_sweep crashed run_id=%d", run_id)
|
||
finally:
|
||
run_db.close()
|
||
|
||
task = asyncio.create_task(_run())
|
||
# Keep reference to avoid GC before task completes (RUF006)
|
||
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
|
||
logger.info(
|
||
"scheduler: triggered city_sweep run_id=%d next_run_at=%s", run_id, next_at.isoformat()
|
||
)
|
||
return run_id
|
||
|
||
|
||
def get_due_schedules(db: Session) -> list[dict[str, Any]]:
|
||
"""SELECT scrape_schedules WHERE enabled AND (next_run_at IS NULL OR next_run_at <= NOW())."""
|
||
rows = db.execute(
|
||
text(
|
||
"""
|
||
SELECT id, source, enabled, window_start_hour, window_end_hour,
|
||
default_params, last_run_id, last_run_at, next_run_at
|
||
FROM scrape_schedules
|
||
WHERE enabled = true
|
||
AND (next_run_at IS NULL OR next_run_at <= NOW())
|
||
"""
|
||
),
|
||
).mappings().all()
|
||
return [dict(r) for r in rows]
|
||
|
||
|
||
async def scheduler_loop() -> None:
|
||
"""Бесконечный async loop — tick каждые SCHEDULER_TICK_SEC секунд."""
|
||
logger.info("scheduler: started (tick=%ds)", SCHEDULER_TICK_SEC)
|
||
# Initial sleep 30s чтобы дать FastAPI startup завершиться
|
||
await asyncio.sleep(30)
|
||
while True:
|
||
try:
|
||
db = SessionLocal()
|
||
try:
|
||
# Reap zombies first
|
||
reap_zombies(db)
|
||
# Process due schedules
|
||
due = get_due_schedules(db)
|
||
for sch in due:
|
||
if sch["source"] == "avito_city_sweep":
|
||
await trigger_avito_city_sweep_run(db, sch)
|
||
else:
|
||
logger.warning("scheduler: unknown source=%s, skip", sch["source"])
|
||
finally:
|
||
db.close()
|
||
except Exception:
|
||
logger.exception("scheduler: tick failed")
|
||
await asyncio.sleep(SCHEDULER_TICK_SEC)
|