124 lines
4 KiB
Python
124 lines
4 KiB
Python
"""Celery app + beat schedule."""
|
||
|
||
import logging
|
||
|
||
from celery import Celery
|
||
from celery.schedules import crontab
|
||
from celery.signals import worker_ready
|
||
|
||
from app.core.config import settings
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
def _parse_cron(spec: str) -> crontab:
|
||
"""Parse 'M H DoM Mon DoW' crontab string into Celery crontab. Empty fields default to '*'."""
|
||
parts = spec.strip().split()
|
||
if len(parts) != 5:
|
||
raise ValueError(f"crontab spec must have 5 fields, got: {spec!r}")
|
||
minute, hour, dom, month, dow = parts
|
||
return crontab(
|
||
minute=minute,
|
||
hour=hour,
|
||
day_of_month=dom,
|
||
month_of_year=month,
|
||
day_of_week=dow,
|
||
)
|
||
|
||
|
||
def _default_regions() -> list[int]:
|
||
return [int(x.strip()) for x in settings.scrape_kn_default_regions.split(",") if x.strip()]
|
||
|
||
|
||
celery_app = Celery(
|
||
"gendesign",
|
||
broker=settings.redis_url,
|
||
backend=settings.redis_url,
|
||
include=["app.workers.tasks.scrape_kn"],
|
||
)
|
||
celery_app.conf.timezone = "Europe/Moscow"
|
||
|
||
# Расписание задаётся через SCRAPE_KN_CRON env var (см. app.core.config).
|
||
# Каждый региональный sweep оборачивается случайной задержкой 0..SCRAPE_KN_JITTER_SECONDS
|
||
# внутри самой задачи (см. tasks.scrape_kn) — чтобы не бить ровно в одну минуту.
|
||
celery_app.conf.beat_schedule = {
|
||
f"kn-region-{rc}": {
|
||
"task": "tasks.scrape_kn.scrape_kn_region",
|
||
"schedule": _parse_cron(settings.scrape_kn_cron),
|
||
"args": [rc, None],
|
||
}
|
||
for rc in _default_regions()
|
||
}
|
||
|
||
|
||
@worker_ready.connect
|
||
def _resume_zombie_runs(sender=None, **_kwargs) -> None:
|
||
"""When a worker finishes booting (after redeploy/restart), find any sweep
|
||
that was 'running' with a stale heartbeat (>5 min) and re-enqueue a resume
|
||
task. Each resume creates a fresh run_id linked via resumed_from_run_id;
|
||
the original row is marked 'zombie' so the audit trail is preserved.
|
||
"""
|
||
from sqlalchemy import text
|
||
|
||
from app.core.db import SessionLocal
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
rows = (
|
||
db.execute(
|
||
text(
|
||
"""
|
||
SELECT run_id
|
||
FROM kn_scrape_runs
|
||
WHERE status = 'running'
|
||
AND objects_snapshot IS NOT NULL
|
||
AND COALESCE(heartbeat_at, started_at)
|
||
< NOW() - INTERVAL '5 minutes'
|
||
ORDER BY started_at ASC
|
||
LIMIT 20
|
||
"""
|
||
)
|
||
)
|
||
.mappings()
|
||
.all()
|
||
)
|
||
if not rows:
|
||
logger.info("worker_ready: нет stale runs для resume")
|
||
return
|
||
|
||
ids = [int(r["run_id"]) for r in rows]
|
||
# Помечаем найденные как 'zombie' одним апдейтом — resume создаст новые
|
||
# run_id со ссылкой resumed_from_run_id.
|
||
db.execute(
|
||
text(
|
||
"""
|
||
UPDATE kn_scrape_runs
|
||
SET status = 'zombie',
|
||
finished_at = NOW(),
|
||
error = COALESCE(error,
|
||
'auto-zombie at worker_ready, resume scheduled')
|
||
WHERE run_id = ANY(:ids)
|
||
"""
|
||
),
|
||
{"ids": ids},
|
||
)
|
||
db.commit()
|
||
except Exception as e:
|
||
logger.exception("worker_ready resume scan failed: %s", e)
|
||
try:
|
||
db.rollback()
|
||
except Exception:
|
||
pass
|
||
return
|
||
finally:
|
||
db.close()
|
||
|
||
# Enqueue resume tasks. Late import to avoid circular at module load.
|
||
from app.workers.tasks.scrape_kn import resume_kn_run
|
||
|
||
for rid in ids:
|
||
try:
|
||
resume_kn_run.apply_async(args=[rid])
|
||
logger.info("worker_ready: resume_kn_run enqueued for run=%s", rid)
|
||
except Exception as e:
|
||
logger.warning("worker_ready: failed to enqueue resume for run=%s: %s", rid, e)
|