From ae86d62e9b4a91ded86159e2157706c206dfed3f Mon Sep 17 00:00:00 2001 From: lekss361 Date: Mon, 11 May 2026 17:12:31 +0300 Subject: [PATCH] fix(worker): worker_ready geo-resume blocked by early return in kn-resume MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of "auto-resume never fired": the kn_scrape_runs resume section hit `if not rows: return` (and similar `return` in except) before reaching the nspd_geo_jobs resume section. Whenever there were no zombie kn-runs (the normal case), the handler bailed out and geo jobs stayed forever 'running' with stale heartbeats — users had to manual cancel/resume after every deploy. Fix: don't return early. Initialize `ids = []`, only run UPDATE if rows exist, drop the inner `return` from exception branch. The for-loop over ids becomes a no-op when empty, and execution falls through to the geo section. Same pattern as the breadcrumb above — fail soft, continue. cleanup_zombies beat task (added in caa467f) stays as belt-and-suspenders in case worker_ready signal ever misbehaves again. --- backend/app/workers/celery_app.py | 46 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 23 deletions(-) diff --git a/backend/app/workers/celery_app.py b/backend/app/workers/celery_app.py index fe1dbc6f..036960fb 100644 --- a/backend/app/workers/celery_app.py +++ b/backend/app/workers/celery_app.py @@ -279,7 +279,9 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None: from app.core.db import SessionLocal + # ── kn_scrape_runs resume ── db = SessionLocal() + ids: list[int] = [] try: rows = ( db.execute( @@ -297,34 +299,32 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None: .mappings() .all() ) - if not rows: - logger.info("worker_ready: нет stale runs для resume") - return - - ids = [int(r["run_id"]) for r in rows] - # Помечаем найденные как 'zombie' одним апдейтом — resume создаст новые - # run_id со ссылкой resumed_from_run_id. - db.execute( - text( - """ - UPDATE kn_scrape_runs - SET status = 'zombie', - finished_at = NOW(), - error = COALESCE(error, - 'auto-zombie at worker_ready, resume scheduled') - WHERE run_id = ANY(:ids) - """ - ), - {"ids": ids}, - ) - db.commit() + if rows: + ids = [int(r["run_id"]) for r in rows] + # Помечаем найденные как 'zombie' одним апдейтом — resume создаст новые + # run_id со ссылкой resumed_from_run_id. + db.execute( + text( + """ + UPDATE kn_scrape_runs + SET status = 'zombie', + finished_at = NOW(), + error = COALESCE(error, + 'auto-zombie at worker_ready, resume scheduled') + WHERE run_id = ANY(:ids) + """ + ), + {"ids": ids}, + ) + db.commit() + else: + logger.info("worker_ready: нет stale kn runs для resume") except Exception as e: - logger.exception("worker_ready resume scan failed: %s", e) + logger.exception("worker_ready kn resume scan failed: %s", e) try: db.rollback() except Exception: pass - return finally: db.close()