fix(worker): worker_ready geo-resume blocked by early return in kn-resume
Root cause of "auto-resume never fired": the kn_scrape_runs resume section
hit `if not rows: return` (and similar `return` in except) before reaching
the nspd_geo_jobs resume section. Whenever there were no zombie kn-runs
(the normal case), the handler bailed out and geo jobs stayed forever
'running' with stale heartbeats — users had to manual cancel/resume after
every deploy.
Fix: don't return early. Initialize `ids = []`, only run UPDATE if rows
exist, drop the inner `return` from exception branch. The for-loop over
ids becomes a no-op when empty, and execution falls through to the geo
section. Same pattern as the breadcrumb above — fail soft, continue.
cleanup_zombies beat task (added in caa467f) stays as belt-and-suspenders
in case worker_ready signal ever misbehaves again.
This commit is contained in:
parent
caa467fb7e
commit
ae86d62e9b
1 changed files with 23 additions and 23 deletions
|
|
@ -279,7 +279,9 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None:
|
|||
|
||||
from app.core.db import SessionLocal
|
||||
|
||||
# ── kn_scrape_runs resume ──
|
||||
db = SessionLocal()
|
||||
ids: list[int] = []
|
||||
try:
|
||||
rows = (
|
||||
db.execute(
|
||||
|
|
@ -297,34 +299,32 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None:
|
|||
.mappings()
|
||||
.all()
|
||||
)
|
||||
if not rows:
|
||||
logger.info("worker_ready: нет stale runs для resume")
|
||||
return
|
||||
|
||||
ids = [int(r["run_id"]) for r in rows]
|
||||
# Помечаем найденные как 'zombie' одним апдейтом — resume создаст новые
|
||||
# run_id со ссылкой resumed_from_run_id.
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE kn_scrape_runs
|
||||
SET status = 'zombie',
|
||||
finished_at = NOW(),
|
||||
error = COALESCE(error,
|
||||
'auto-zombie at worker_ready, resume scheduled')
|
||||
WHERE run_id = ANY(:ids)
|
||||
"""
|
||||
),
|
||||
{"ids": ids},
|
||||
)
|
||||
db.commit()
|
||||
if rows:
|
||||
ids = [int(r["run_id"]) for r in rows]
|
||||
# Помечаем найденные как 'zombie' одним апдейтом — resume создаст новые
|
||||
# run_id со ссылкой resumed_from_run_id.
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE kn_scrape_runs
|
||||
SET status = 'zombie',
|
||||
finished_at = NOW(),
|
||||
error = COALESCE(error,
|
||||
'auto-zombie at worker_ready, resume scheduled')
|
||||
WHERE run_id = ANY(:ids)
|
||||
"""
|
||||
),
|
||||
{"ids": ids},
|
||||
)
|
||||
db.commit()
|
||||
else:
|
||||
logger.info("worker_ready: нет stale kn runs для resume")
|
||||
except Exception as e:
|
||||
logger.exception("worker_ready resume scan failed: %s", e)
|
||||
logger.exception("worker_ready kn resume scan failed: %s", e)
|
||||
try:
|
||||
db.rollback()
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue