fix(worker): worker_ready geo-resume blocked by early return in kn-resume

Root cause of "auto-resume never fired": the kn_scrape_runs resume section
hit `if not rows: return` (and similar `return` in except) before reaching
the nspd_geo_jobs resume section. Whenever there were no zombie kn-runs
(the normal case), the handler bailed out and geo jobs stayed forever
'running' with stale heartbeats — users had to manual cancel/resume after
every deploy.

Fix: don't return early. Initialize `ids = []`, only run UPDATE if rows
exist, drop the inner `return` from exception branch. The for-loop over
ids becomes a no-op when empty, and execution falls through to the geo
section. Same pattern as the breadcrumb above — fail soft, continue.

cleanup_zombies beat task (added in caa467f) stays as belt-and-suspenders
in case worker_ready signal ever misbehaves again.
This commit is contained in:
lekss361 2026-05-11 17:12:31 +03:00
parent caa467fb7e
commit ae86d62e9b

View file

@ -279,7 +279,9 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None:
from app.core.db import SessionLocal from app.core.db import SessionLocal
# ── kn_scrape_runs resume ──
db = SessionLocal() db = SessionLocal()
ids: list[int] = []
try: try:
rows = ( rows = (
db.execute( db.execute(
@ -297,34 +299,32 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None:
.mappings() .mappings()
.all() .all()
) )
if not rows: if rows:
logger.info("worker_ready: нет stale runs для resume") ids = [int(r["run_id"]) for r in rows]
return # Помечаем найденные как 'zombie' одним апдейтом — resume создаст новые
# run_id со ссылкой resumed_from_run_id.
ids = [int(r["run_id"]) for r in rows] db.execute(
# Помечаем найденные как 'zombie' одним апдейтом — resume создаст новые text(
# run_id со ссылкой resumed_from_run_id. """
db.execute( UPDATE kn_scrape_runs
text( SET status = 'zombie',
""" finished_at = NOW(),
UPDATE kn_scrape_runs error = COALESCE(error,
SET status = 'zombie', 'auto-zombie at worker_ready, resume scheduled')
finished_at = NOW(), WHERE run_id = ANY(:ids)
error = COALESCE(error, """
'auto-zombie at worker_ready, resume scheduled') ),
WHERE run_id = ANY(:ids) {"ids": ids},
""" )
), db.commit()
{"ids": ids}, else:
) logger.info("worker_ready: нет stale kn runs для resume")
db.commit()
except Exception as e: except Exception as e:
logger.exception("worker_ready resume scan failed: %s", e) logger.exception("worker_ready kn resume scan failed: %s", e)
try: try:
db.rollback() db.rollback()
except Exception: except Exception:
pass pass
return
finally: finally:
db.close() db.close()