fix(worker): worker_ready geo-resume blocked by early return in kn-resume
Root cause of "auto-resume never fired": the kn_scrape_runs resume section
hit `if not rows: return` (and similar `return` in except) before reaching
the nspd_geo_jobs resume section. Whenever there were no zombie kn-runs
(the normal case), the handler bailed out and geo jobs stayed forever
'running' with stale heartbeats — users had to manual cancel/resume after
every deploy.
Fix: don't return early. Initialize `ids = []`, only run UPDATE if rows
exist, drop the inner `return` from exception branch. The for-loop over
ids becomes a no-op when empty, and execution falls through to the geo
section. Same pattern as the breadcrumb above — fail soft, continue.
cleanup_zombies beat task (added in caa467f) stays as belt-and-suspenders
in case worker_ready signal ever misbehaves again.
This commit is contained in:
parent
caa467fb7e
commit
ae86d62e9b
1 changed files with 23 additions and 23 deletions
|
|
@ -279,7 +279,9 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None:
|
||||||
|
|
||||||
from app.core.db import SessionLocal
|
from app.core.db import SessionLocal
|
||||||
|
|
||||||
|
# ── kn_scrape_runs resume ──
|
||||||
db = SessionLocal()
|
db = SessionLocal()
|
||||||
|
ids: list[int] = []
|
||||||
try:
|
try:
|
||||||
rows = (
|
rows = (
|
||||||
db.execute(
|
db.execute(
|
||||||
|
|
@ -297,34 +299,32 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None:
|
||||||
.mappings()
|
.mappings()
|
||||||
.all()
|
.all()
|
||||||
)
|
)
|
||||||
if not rows:
|
if rows:
|
||||||
logger.info("worker_ready: нет stale runs для resume")
|
ids = [int(r["run_id"]) for r in rows]
|
||||||
return
|
# Помечаем найденные как 'zombie' одним апдейтом — resume создаст новые
|
||||||
|
# run_id со ссылкой resumed_from_run_id.
|
||||||
ids = [int(r["run_id"]) for r in rows]
|
db.execute(
|
||||||
# Помечаем найденные как 'zombie' одним апдейтом — resume создаст новые
|
text(
|
||||||
# run_id со ссылкой resumed_from_run_id.
|
"""
|
||||||
db.execute(
|
UPDATE kn_scrape_runs
|
||||||
text(
|
SET status = 'zombie',
|
||||||
"""
|
finished_at = NOW(),
|
||||||
UPDATE kn_scrape_runs
|
error = COALESCE(error,
|
||||||
SET status = 'zombie',
|
'auto-zombie at worker_ready, resume scheduled')
|
||||||
finished_at = NOW(),
|
WHERE run_id = ANY(:ids)
|
||||||
error = COALESCE(error,
|
"""
|
||||||
'auto-zombie at worker_ready, resume scheduled')
|
),
|
||||||
WHERE run_id = ANY(:ids)
|
{"ids": ids},
|
||||||
"""
|
)
|
||||||
),
|
db.commit()
|
||||||
{"ids": ids},
|
else:
|
||||||
)
|
logger.info("worker_ready: нет stale kn runs для resume")
|
||||||
db.commit()
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.exception("worker_ready resume scan failed: %s", e)
|
logger.exception("worker_ready kn resume scan failed: %s", e)
|
||||||
try:
|
try:
|
||||||
db.rollback()
|
db.rollback()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
return
|
|
||||||
finally:
|
finally:
|
||||||
db.close()
|
db.close()
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue