fix(tradein/scraper): drain detached run-children on SIGTERM, not just coordinator (#1182 Phase 2)
All checks were successful
CI / changes (pull_request) Successful in 6s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped

Pre-push review: _await_scheduler ждал только scheduler_loop COORDINATOR, но вся
scrape-работа крутится в detached asyncio.create_task детях (каждый trigger_* делал
`task = create_task(_run())` без join). На SIGTERM coordinator выходил из while True
и завершался → asyncio.run() teardown хард-кансельил ещё бегущих детей mid-await =
ровно #1182 failure mode. Кооперативный checkpoint спасал ребёнка лишь когда его
residual-время случайно перекрывало drain — вероятностно, не гарантированно.

Fix — coordinator теперь дренажит детей перед выходом:
- _spawn_tracked(coro): централизованный detached-spawn, кладёт задачу в module-level
  registry _inflight_tasks (strong-ref = RUF006 keep-alive) + done-callback ретривит
  exception и убирает из set'а. Заменил 26 одинаковых
  `task = create_task(_run()); task.add_done_callback(...)` сайтов.
- _drain_inflight(): один asyncio.wait по живым детям с бюджетом
  _CHILD_DRAIN_TIMEOUT_S=80s (< scheduler_main 100s < docker grace 120s). Кооперативные
  дети (avito_detail_backfill, rosreestr-executor) дочекивают карточку/батч + mark_done
  и резолвятся; некооперативные упираются в timeout и падают на внешний hard-cancel.
- scheduler_loop по выходу из tick-loop (только по SIGTERM) зовёт await _drain_inflight().

NB: raw asyncio.all_tasks()-minus-self здесь НЕЛЬЗЯ — в нашей топологии он захватывает
_run parent-task (блокирован на wait_for(coordinator)) и shutdown_waiter → циклическое
ожидание coordinator↔_run, всегда упирающееся в timeout. Точный registry это исключает.

Tests: tests/test_scheduler.py — detached cooperative child drained-not-cancelled,
non-cooperative child timeout→left for hard-cancel, no-op без детей, scheduler_loop→drain
wiring. Обновил 3 source-inspection теста под новый _spawn_tracked паттерн.
This commit is contained in:
bot-backend 2026-06-28 20:13:53 +03:00
parent 6f76b565e5
commit 566b2f9617
5 changed files with 197 additions and 69 deletions

View file

@ -132,6 +132,7 @@ from __future__ import annotations
import asyncio
import logging
import random
from collections.abc import Coroutine
from datetime import UTC, datetime, time, timedelta
from typing import Any
@ -158,6 +159,63 @@ logger = logging.getLogger(__name__)
SCHEDULER_TICK_SEC = 60
ZOMBIE_THRESHOLD_HOURS = 6
# #1182 Phase 2: graceful SIGTERM-drain. Scrape-работа крутится в detached
# asyncio.create_task (см. _spawn_tracked) — coordinator (scheduler_loop) их
# НЕ join'ит. Без явного дренажа asyncio.run() teardown хард-кансельнул бы их
# mid-await (тот самый #1182 failure mode). Регистр strong-ref'ов даёт coordinator'у
# дождаться детей перед выходом из tick-loop'а.
_inflight_tasks: set[asyncio.Task[None]] = set()
# Бюджет дренажа детей < scheduler_main._DRAIN_TIMEOUT_S (100s) < docker grace (120s):
# coordinator успевает дождаться кооперативных детей (avito_detail_backfill,
# rosreestr-executor дочекивают карточку/батч + mark_done и резолвятся) ВНУТРИ
# внешнего wait_for, а некооперативные упираются в этот timeout и падают на внешний
# hard-cancel fallback.
_CHILD_DRAIN_TIMEOUT_S = 80.0
def _spawn_tracked(coro: Coroutine[Any, Any, None]) -> asyncio.Task[None]:
"""create_task + регистрация задачи в _inflight_tasks для graceful-drain'а (#1182 P2).
Заменяет прежний `task = create_task(_run()); task.add_done_callback(...)`:
strong-ref в set'е держит задачу до завершения (RUF006) и даёт scheduler_loop'у
дождаться её на SIGTERM-drain'е; done-callback ретривит exception (чтобы не было
'Task exception was never retrieved') и убирает задачу из set'а.
"""
task = asyncio.create_task(coro)
_inflight_tasks.add(task)
def _on_done(t: asyncio.Task[None]) -> None:
_inflight_tasks.discard(t)
if not t.cancelled():
t.exception()
task.add_done_callback(_on_done)
return task
async def _drain_inflight() -> None:
"""Дождаться завершения detached run-задач перед teardown'ом процесса (#1182 P2).
Зовётся из scheduler_loop при выходе из tick-loop'а по SIGTERM-drain'у. Кооперативные
дети дочекивают текущий unit (карточку/батч), делают mark_done и резолвятся;
некооперативные упираются в _CHILD_DRAIN_TIMEOUT_S и остаются на внешний hard-cancel
fallback (scheduler_main 100s + docker 120s). Не busy-spin один asyncio.wait.
"""
pending = [t for t in _inflight_tasks if not t.done()]
if not pending:
return
logger.info("scheduler: draining %d in-flight run task(s) on shutdown", len(pending))
_done, still = await asyncio.wait(pending, timeout=_CHILD_DRAIN_TIMEOUT_S)
if still:
logger.warning(
"scheduler: %d task(s) did not drain in %.0fs — leaving for hard-cancel",
len(still),
_CHILD_DRAIN_TIMEOUT_S,
)
else:
logger.info("scheduler: all in-flight run task(s) drained cleanly")
def compute_next_run_at(
window_start_hour: int,
@ -369,9 +427,7 @@ async def trigger_avito_city_sweep_run(db: Session, schedule_row: dict[str, Any]
finally:
run_db.close()
task = asyncio.create_task(_run())
# Keep reference to avoid GC before task completes (RUF006)
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered city_sweep run_id=%d", run_id)
return run_id
@ -407,9 +463,7 @@ async def trigger_avito_newbuilding_sweep_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
# Keep reference to avoid GC before task completes (RUF006)
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered newbuilding_sweep run_id=%d", run_id)
return run_id
@ -447,9 +501,7 @@ async def trigger_yandex_city_sweep_run(db: Session, schedule_row: dict[str, Any
finally:
run_db.close()
task = asyncio.create_task(_run())
# Keep reference to avoid GC before task completes (RUF006)
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered yandex_city_sweep run_id=%d", run_id)
return run_id
@ -507,8 +559,7 @@ async def trigger_cian_backfill_run(db: Session, schedule_row: dict[str, Any]) -
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered cian_history_backfill run_id=%d", run_id)
return run_id
@ -543,8 +594,7 @@ async def trigger_cian_city_sweep_run(db: Session, schedule_row: dict[str, Any])
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered cian_city_sweep run_id=%d", run_id)
return run_id
@ -579,8 +629,7 @@ async def trigger_cian_full_load_run(db: Session, schedule_row: dict[str, Any])
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered cian_full_load run_id=%d", run_id)
return run_id
@ -625,8 +674,7 @@ async def trigger_avito_full_load_run(db: Session, schedule_row: dict[str, Any])
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered avito_full_load run_id=%d", run_id)
return run_id
@ -669,8 +717,7 @@ async def trigger_avito_full_load_exhaustive_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered avito_full_load_exhaustive run_id=%d", run_id)
return run_id
@ -706,8 +753,7 @@ async def trigger_domclick_city_sweep_run(db: Session, schedule_row: dict[str, A
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered domclick_city_sweep run_id=%d", run_id)
return run_id
@ -810,8 +856,7 @@ async def trigger_yandex_address_backfill_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered yandex_address_backfill run_id=%d", run_id)
return run_id
@ -849,8 +894,7 @@ async def trigger_geocode_missing_listings_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered geocode_missing_listings run_id=%d", run_id)
return run_id
@ -874,8 +918,7 @@ async def trigger_rosreestr_dkp_run(db: Session, schedule_row: dict[str, Any]) -
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered rosreestr_dkp_import run_id=%d", run_id)
return run_id
@ -909,8 +952,7 @@ async def trigger_listing_source_snapshot_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered listing_source_snapshot run_id=%d", run_id)
return run_id
@ -954,8 +996,7 @@ async def trigger_refresh_search_matview_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered refresh_search_matview run_id=%d", run_id)
return run_id
@ -989,8 +1030,7 @@ async def trigger_deactivate_stale_avito_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered deactivate_stale_avito run_id=%d", run_id)
return run_id
@ -1042,8 +1082,7 @@ async def trigger_deactivate_stale_run(db: Session, schedule_row: dict[str, Any]
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered deactivate_stale source=%s run_id=%d", listing_source, run_id)
return run_id
@ -1078,8 +1117,7 @@ async def trigger_sber_index_pull_run(db: Session, schedule_row: dict[str, Any])
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered sber_index_pull run_id=%d", run_id)
return run_id
@ -1116,8 +1154,7 @@ async def trigger_rosreestr_quarter_poll_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered rosreestr_quarter_poll run_id=%d", run_id)
return run_id
@ -1159,8 +1196,7 @@ async def trigger_newbuilding_enrich_run(db: Session, schedule_row: dict[str, An
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered newbuilding_enrich run_id=%d", run_id)
return run_id
@ -1208,8 +1244,7 @@ async def trigger_yandex_newbuilding_sweep_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered yandex_newbuilding_sweep run_id=%d", run_id)
return run_id
@ -1243,8 +1278,7 @@ async def trigger_asking_to_sold_ratio_run(db: Session, schedule_row: dict[str,
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered asking_to_sold_ratio_refresh run_id=%d", run_id)
return run_id
@ -1531,8 +1565,7 @@ async def trigger_avito_detail_backfill_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered avito_detail_backfill run_id=%d", run_id)
return run_id
@ -1568,8 +1601,7 @@ async def trigger_yandex_detail_backfill_run(
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered yandex_detail_backfill run_id=%d", run_id)
return run_id
@ -1610,8 +1642,7 @@ async def trigger_cadastral_geo_match_run(db: Session, schedule_row: dict[str, A
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered cadastral_geo_match run_id=%d", run_id)
return run_id
@ -1683,8 +1714,7 @@ async def trigger_house_imv_backfill_run(db: Session, schedule_row: dict[str, An
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered house_imv_backfill run_id=%d", run_id)
return run_id
@ -1730,8 +1760,7 @@ async def trigger_house_dedup_merge_run(db: Session, schedule_row: dict[str, Any
finally:
run_db.close()
task = asyncio.create_task(_run())
task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
_spawn_tracked(_run())
logger.info("scheduler: triggered house_dedup_merge run_id=%d", run_id)
return run_id
@ -1846,3 +1875,9 @@ async def scheduler_loop() -> None:
logger.info("scheduler: SIGTERM-drain — exiting tick loop after dispatch")
break
await asyncio.sleep(SCHEDULER_TICK_SEC)
# Tick-loop вышел только по SIGTERM-drain'у (иначе while True бесконечен): дожидаемся
# detached run-задач (_spawn_tracked) — пусть докоммитят текущий unit и сделают
# mark_done, а не дадим asyncio.run() teardown'у хард-кансельнуть их mid-await (#1182).
await _drain_inflight()
logger.info("scheduler: tick loop exited (in-flight drain complete)")

View file

@ -4,6 +4,7 @@ No real network, no real Postgres. The Cian fetch is mocked; the DB session is a
lightweight in-memory fake that models just enough of the three target tables to
assert (a) rows land and (b) an idempotent re-run does not duplicate them.
"""
from __future__ import annotations
import os
@ -636,7 +637,9 @@ def test_trigger_claims_run_and_delegates_to_task() -> None:
trig_src = inspect.getsource(scheduler.trigger_newbuilding_enrich_run)
assert "_claim_run(db, schedule_row)" in trig_src
assert "run_newbuilding_enrich" in trig_src
assert "asyncio.create_task(" in trig_src
# #1182 P2: detached spawn централизован в _spawn_tracked (внутри — asyncio.create_task
# + registry strong-ref для graceful SIGTERM-drain).
assert "_spawn_tracked(_run())" in trig_src
@pytest.mark.asyncio

View file

@ -396,9 +396,10 @@ def test_trigger_claims_run_and_runs_in_executor() -> None:
# sync DB-only task → run_in_executor (mirror cadastral_geo_match, not a bare await).
assert "run_in_executor" in src
assert "run_house_dedup_merge" in src
# fresh session inside the spawned task + RUF006 keep-alive callback.
# fresh session inside the spawned task + #1182 P2 detached spawn via _spawn_tracked
# (registry strong-ref = RUF006 keep-alive + graceful SIGTERM-drain).
assert "run_db = SessionLocal()" in src
assert "task.add_done_callback(" in src
assert "_spawn_tracked(_run())" in src
assert "run_db.close()" in src
@ -535,9 +536,7 @@ def test_real_merge_repoints_dedups_deletes_and_is_idempotent() -> None:
).all()
assert [r.id for r in survivors] == [900001]
# loser's listing re-pointed to keeper.
repointed = db.execute(
_t("SELECT house_id_fk FROM listings WHERE id = 910002")
).scalar()
repointed = db.execute(_t("SELECT house_id_fk FROM listings WHERE id = 910002")).scalar()
assert repointed == 900001
# price_dynamics deduped — exactly one row for the keeper at that 6-col key (no dup),
# and it is the KEEPER's own row (price_per_sqm=100000), not the loser's (999999):
@ -580,10 +579,7 @@ def test_real_merge_repoints_dedups_deletes_and_is_idempotent() -> None:
_t("DELETE FROM house_sources WHERE ext_id IN ('SRC-KEEP','SRC-LOSE','EXT-KEEP')")
)
db.execute(
_t(
"DELETE FROM house_address_aliases "
"WHERE normalized_address = 'тестдом 1772, 1'"
)
_t("DELETE FROM house_address_aliases WHERE normalized_address = 'тестдом 1772, 1'")
)
db.execute(_t("DELETE FROM houses WHERE id IN (900001,900002)"))
db.commit()

View file

@ -54,9 +54,10 @@ def test_trigger_mirrors_backfill_sibling_pattern() -> None:
assert "backfill_house_imv" in _TRIGGER_SRC
# async service → bare await (NOT run_in_executor, unlike the sync DB-only triggers).
assert "run_in_executor" not in _TRIGGER_SRC
# RUF006: keep a reference + done-callback so the task is not GC'd mid-flight.
assert "task = asyncio.create_task(_run())" in _TRIGGER_SRC
assert "task.add_done_callback(" in _TRIGGER_SRC
# #1182 P2: detached spawn через _spawn_tracked — strong-ref в registry (RUF006 +
# graceful SIGTERM-drain), done-callback ретривит exception. Заменил прежний
# `task = asyncio.create_task(_run()); task.add_done_callback(...)`.
assert "_spawn_tracked(_run())" in _TRIGGER_SRC
assert "run_db.close()" in _TRIGGER_SRC

View file

@ -2,7 +2,7 @@
import asyncio
import os
from unittest.mock import MagicMock
from unittest.mock import AsyncMock, MagicMock
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
@ -10,9 +10,22 @@ from datetime import UTC, datetime
import pytest
import app.services.scheduler as sched
from app.core import shutdown as sd
from app.services.scheduler import ZOMBIE_THRESHOLD_HOURS, compute_next_run_at
@pytest.fixture(autouse=True)
def _reset_drain_state() -> None:
"""#1182 P2: shutdown-флаг и registry детей — module-global; чистим вокруг каждого
теста (изоляция от detached-задач, что триггеры спавнят в _inflight_tasks)."""
sd.reset_shutdown()
sched._inflight_tasks.clear()
yield
sched._inflight_tasks.clear()
sd.reset_shutdown()
def test_compute_next_run_in_normal_window() -> None:
now = datetime(2026, 5, 23, 12, 0, tzinfo=UTC)
next_at = compute_next_run_at(2, 5, now=now)
@ -730,3 +743,83 @@ async def test_scheduler_dispatch_routes_avito_full_load_exhaustive(
await _sched.trigger_avito_full_load_exhaustive_run(db, sch)
assert "avito_full_load_exhaustive" in triggered
# ---------------------------------------------------------------------------
# #1182 Phase 2 — graceful SIGTERM-drain детей (detached run-задач)
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_drain_inflight_awaits_detached_cooperative_child() -> None:
"""Гвоздь Phase 2: _drain_inflight ДОЖИДАЕТСЯ detached-ребёнка (как scrape-run,
отвязанный через _spawn_tracked) до его clean-exit, а НЕ хард-кансельит его.
Регрессия gap'а: coordinator раньше ждал только себя → asyncio.run teardown
хард-кансельил живых детей mid-await (#1182 failure mode).
"""
clean_exit = asyncio.Event() # == mark_done-эквивалент ребёнка
async def _child() -> None:
# Кооперативный ребёнок: крутится, пока не запрошен shutdown, потом выходит чисто.
while not sd.shutdown_requested():
await asyncio.sleep(0.01)
clean_exit.set()
child = sched._spawn_tracked(_child()) # detached, попадает в _inflight_tasks
await asyncio.sleep(0.02)
assert not child.done(), "ребёнок ещё бежит (shutdown не запрошен)"
sd.request_shutdown()
await asyncio.wait_for(sched._drain_inflight(), timeout=2.0)
assert clean_exit.is_set(), "drain дождался clean-exit ребёнка"
assert child.done()
assert not child.cancelled(), "ребёнок вышел сам, НЕ хард-cancel"
@pytest.mark.asyncio
async def test_drain_inflight_times_out_on_noncooperative_child(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Некооперативный ребёнок (игнорит shutdown) упирается в _CHILD_DRAIN_TIMEOUT_S и
остаётся бежать отдаётся внешнему hard-cancel fallback (scheduler_main/docker)."""
monkeypatch.setattr(sched, "_CHILD_DRAIN_TIMEOUT_S", 0.05)
async def _stuck() -> None:
await asyncio.Event().wait() # никогда не проверяет shutdown_requested()
child = sched._spawn_tracked(_stuck())
sd.request_shutdown()
await asyncio.wait_for(sched._drain_inflight(), timeout=2.0)
assert not child.done(), "drain НЕ повесил процесс — оставил ребёнка под hard-cancel"
# cleanup
child.cancel()
with pytest.raises(asyncio.CancelledError):
await child
@pytest.mark.asyncio
async def test_drain_inflight_noop_when_no_children() -> None:
"""Без in-flight детей _drain_inflight() — мгновенный no-op (без сна/ожидания)."""
assert not sched._inflight_tasks
await asyncio.wait_for(sched._drain_inflight(), timeout=1.0)
@pytest.mark.asyncio
async def test_scheduler_loop_calls_drain_on_shutdown(monkeypatch: pytest.MonkeyPatch) -> None:
"""scheduler_loop при выходе по SIGTERM-drain'у вызывает _drain_inflight (проводка
coordinator дренаж детей). Initial sleep(30) и tick-sleep занулены, чтобы тест
был детерминированным и быстрым."""
drain_spy = AsyncMock()
monkeypatch.setattr(sched, "_drain_inflight", drain_spy)
monkeypatch.setattr(sched.asyncio, "sleep", AsyncMock()) # без реальных 30s/60s
sd.request_shutdown() # первый же top-of-tick check → break
await asyncio.wait_for(sched.scheduler_loop(), timeout=2.0)
drain_spy.assert_awaited_once()