feat(scraper-kit): copy scheduler with registry-dispatch (25+ trigger_* → dict), strangler (#2136)
This commit is contained in:
parent
abdb413a99
commit
0e272188e2
2 changed files with 1165 additions and 0 deletions
471
tradein-mvp/backend/tests/test_scraper_kit_scheduler_parity.py
Normal file
471
tradein-mvp/backend/tests/test_scraper_kit_scheduler_parity.py
Normal file
|
|
@ -0,0 +1,471 @@
|
|||
"""Golden-parity: `scraper_kit.orchestration.scheduler` registry-dispatch ≡ старый if/elif.
|
||||
|
||||
Strangler-инвариант (#2136): registry-реестр (`build_registry` + `resolve_handler` +
|
||||
`_dispatch`) должен маршрутизировать те же source'ы к тем же обработчикам, что и боевой
|
||||
27-веточный `if/elif` в `app.services.scheduler.scheduler_loop`, и сохранить критичную
|
||||
concurrency-логику (`_claim_run` advisory-lock + double-check, `reap_zombies` порог,
|
||||
proxy_healthcheck sub-hourly post-claim).
|
||||
|
||||
Метод:
|
||||
1. ROUTING: для каждого source прогоняем ОДИН тик боевого `scheduler_loop` (все
|
||||
`trigger_*` замоканы) → фиксируем какой trigger сработал; параллельно `resolve_handler`
|
||||
kit-реестра обязан вернуть handler для того же source. Множества покрытых source'ов
|
||||
обязаны совпасть → таблица маршрутизации эквивалентна.
|
||||
2. CLAIM: `_claim_run` — happy-path, already-running skip, lock-busy skip, appeared-under-
|
||||
lock rollback. Мокаем БД + ctx.runs.
|
||||
3. ZOMBIE: `reap_zombies` возвращает число + commit.
|
||||
4. SUB-HOURLY: `reschedule_after_minutes` post_claim UPDATE + commit (proxy_healthcheck).
|
||||
5. DISPATCH e2e: `_dispatch` для kit-native sweep (мок pipeline) и продуктового handler —
|
||||
claim → свежая сессия → job вызван → close.
|
||||
|
||||
Без сети, без БД.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from contextlib import ExitStack
|
||||
from typing import Any
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||||
|
||||
from scraper_kit.orchestration import scheduler as kit_sched
|
||||
from scraper_kit.orchestration.scheduler import (
|
||||
Handler,
|
||||
SchedulerContext,
|
||||
_claim_run,
|
||||
_dispatch,
|
||||
build_registry,
|
||||
reap_zombies,
|
||||
reschedule_after_minutes,
|
||||
resolve_handler,
|
||||
)
|
||||
|
||||
import app.services.scheduler as old_sched
|
||||
|
||||
# ── таблица маршрутизации: source → имя боевого trigger_* ─────────────────────
|
||||
# (то, что `if/elif` в старом scheduler_loop выбирает для каждого source).
|
||||
SOURCE_TO_OLD_TRIGGER: dict[str, str] = {
|
||||
"avito_city_sweep": "trigger_avito_city_sweep_run",
|
||||
"avito_full_load": "trigger_avito_full_load_run",
|
||||
"avito_full_load_exhaustive": "trigger_avito_full_load_exhaustive_run",
|
||||
"avito_newbuilding_sweep": "trigger_avito_newbuilding_sweep_run",
|
||||
"yandex_city_sweep": "trigger_yandex_city_sweep_run",
|
||||
"cian_history_backfill": "trigger_cian_backfill_run",
|
||||
"cian_city_sweep": "trigger_cian_city_sweep_run",
|
||||
"cian_full_load": "trigger_cian_full_load_run",
|
||||
"domclick_city_sweep": "trigger_domclick_city_sweep_run",
|
||||
"rosreestr_dkp_import": "trigger_rosreestr_dkp_run",
|
||||
"listing_source_snapshot": "trigger_listing_source_snapshot_run",
|
||||
"asking_to_sold_ratio_refresh": "trigger_asking_to_sold_ratio_run",
|
||||
"refresh_search_matview": "trigger_refresh_search_matview_run",
|
||||
"yandex_address_backfill": "trigger_yandex_address_backfill_run",
|
||||
"deactivate_stale_avito": "trigger_deactivate_stale_run",
|
||||
"deactivate_stale_yandex": "trigger_deactivate_stale_run",
|
||||
"deactivate_stale_cian": "trigger_deactivate_stale_run",
|
||||
"sber_index_pull": "trigger_sber_index_pull_run",
|
||||
"rosreestr_quarter_poll": "trigger_rosreestr_quarter_poll_run",
|
||||
"newbuilding_enrich": "trigger_newbuilding_enrich_run",
|
||||
"yandex_newbuilding_sweep": "trigger_yandex_newbuilding_sweep_run",
|
||||
"geocode_missing_listings": "trigger_geocode_missing_listings_run",
|
||||
"avito_detail_backfill": "trigger_avito_detail_backfill_run",
|
||||
"yandex_detail_backfill": "trigger_yandex_detail_backfill_run",
|
||||
"cadastral_geo_match": "trigger_cadastral_geo_match_run",
|
||||
"house_imv_backfill": "trigger_house_imv_backfill_run",
|
||||
"house_dedup_merge": "trigger_house_dedup_merge_run",
|
||||
"proxy_healthcheck": "trigger_proxy_healthcheck_run",
|
||||
}
|
||||
|
||||
# все distinct trigger-имена, которые надо замокать в боевом scheduler_loop
|
||||
_ALL_TRIGGER_NAMES = sorted(set(SOURCE_TO_OLD_TRIGGER.values()))
|
||||
|
||||
# kit-native (тело в scraper_kit.orchestration.pipeline) — регистрируются встроенно
|
||||
_KIT_NATIVE_SOURCES = {
|
||||
"avito_city_sweep",
|
||||
"avito_full_load",
|
||||
"avito_full_load_exhaustive",
|
||||
"avito_newbuilding_sweep",
|
||||
"yandex_city_sweep",
|
||||
"cian_city_sweep",
|
||||
"cian_full_load",
|
||||
"domclick_city_sweep",
|
||||
}
|
||||
|
||||
|
||||
def _make_sched(source: str) -> dict[str, Any]:
|
||||
return {
|
||||
"id": 1,
|
||||
"source": source,
|
||||
"enabled": True,
|
||||
"window_start_hour": 2,
|
||||
"window_end_hour": 5,
|
||||
"default_params": {},
|
||||
"last_run_id": None,
|
||||
"last_run_at": None,
|
||||
"next_run_at": None,
|
||||
}
|
||||
|
||||
|
||||
# ── kit-реестр с продуктовыми handler'ами (recording stubs) ──────────────────
|
||||
|
||||
|
||||
def _build_recording_registry() -> tuple[dict[str, Handler], dict[str, MagicMock]]:
|
||||
"""Kit-native sweeps + продуктовые recording-handler'ы. Возвращает (registry, fired-map)."""
|
||||
fired: dict[str, MagicMock] = {}
|
||||
|
||||
def _make_job(name: str) -> Any:
|
||||
rec = MagicMock(name=name)
|
||||
fired[name] = rec
|
||||
|
||||
async def _job(db: Any, run_id: int, params: dict[str, Any], ctx: Any) -> None:
|
||||
rec(db, run_id, params)
|
||||
|
||||
return _job
|
||||
|
||||
product: dict[str, Handler] = {}
|
||||
# продуктовые exact-match источники
|
||||
for src in SOURCE_TO_OLD_TRIGGER:
|
||||
if src in _KIT_NATIVE_SOURCES or src.startswith("deactivate_stale_"):
|
||||
continue
|
||||
product[src] = Handler(_make_job(src), src)
|
||||
# deactivate_stale_* — одна wildcard-запись (как боевой startswith)
|
||||
product["deactivate_stale_*"] = Handler(_make_job("deactivate_stale"), "deactivate_stale")
|
||||
|
||||
return build_registry(product), fired
|
||||
|
||||
|
||||
# ── 1. ROUTING parity ────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def _drive_old_one_tick(source: str) -> list[str]:
|
||||
"""Прогнать ОДИН тик боевого scheduler_loop для одного source, вернуть сработавшие trigger'ы."""
|
||||
sch = _make_sched(source)
|
||||
triggers = {name: AsyncMock(return_value=1) for name in _ALL_TRIGGER_NAMES}
|
||||
|
||||
with ExitStack() as es:
|
||||
es.enter_context(patch.object(old_sched, "reap_zombies", MagicMock()))
|
||||
es.enter_context(
|
||||
patch.object(old_sched, "get_due_schedules", MagicMock(return_value=[sch]))
|
||||
)
|
||||
es.enter_context(patch.object(old_sched, "SessionLocal", MagicMock()))
|
||||
es.enter_context(patch.object(old_sched.asyncio, "sleep", AsyncMock()))
|
||||
# shutdown: False на входе while, затем True после первого dispatch → выход
|
||||
es.enter_context(
|
||||
patch.object(
|
||||
old_sched, "shutdown_requested", MagicMock(side_effect=[False] + [True] * 10)
|
||||
)
|
||||
)
|
||||
for name, mock in triggers.items():
|
||||
es.enter_context(patch.object(old_sched, name, mock))
|
||||
await old_sched.scheduler_loop()
|
||||
|
||||
return [name for name, m in triggers.items() if m.await_count > 0]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("source", sorted(SOURCE_TO_OLD_TRIGGER))
|
||||
async def test_routing_parity_per_source(source: str) -> None:
|
||||
"""Для каждого source: старый if/elif зовёт ожидаемый trigger, kit-реестр резолвит handler."""
|
||||
expected_trigger = SOURCE_TO_OLD_TRIGGER[source]
|
||||
|
||||
# старый роутинг
|
||||
fired = await _drive_old_one_tick(source)
|
||||
assert fired == [expected_trigger], (
|
||||
f"source={source}: old scheduler fired {fired}, expected [{expected_trigger}]"
|
||||
)
|
||||
|
||||
# kit роутинг — тот же source обязан резолвиться в handler
|
||||
registry, _ = _build_recording_registry()
|
||||
handler = resolve_handler(source, registry)
|
||||
assert handler is not None, f"kit registry has no handler for source={source}"
|
||||
|
||||
|
||||
def test_routing_coverage_sets_match() -> None:
|
||||
"""Множество source'ов kit-реестра ⊇ множество, покрытое боевым if/elif."""
|
||||
registry, _ = _build_recording_registry()
|
||||
for source in SOURCE_TO_OLD_TRIGGER:
|
||||
assert resolve_handler(source, registry) is not None, f"kit misses source={source}"
|
||||
|
||||
|
||||
def test_kit_native_handler_set() -> None:
|
||||
"""Ровно 8 kit-native sweep-обработчиков зарегистрированы встроенно."""
|
||||
registry = build_registry()
|
||||
assert set(registry) == _KIT_NATIVE_SOURCES
|
||||
for src in _KIT_NATIVE_SOURCES:
|
||||
assert resolve_handler(src, registry).log_name == src
|
||||
|
||||
|
||||
def test_unknown_source_resolves_none() -> None:
|
||||
registry, _ = _build_recording_registry()
|
||||
assert resolve_handler("totally_unknown_source", registry) is None
|
||||
|
||||
|
||||
def test_kit_scheduler_has_no_app_imports() -> None:
|
||||
"""kit scheduler развязан от app.* — ни одного `import app` / `from app` (AST, не substring)."""
|
||||
import ast
|
||||
import inspect
|
||||
|
||||
source = inspect.getsource(kit_sched)
|
||||
tree = ast.parse(source)
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Import):
|
||||
for alias in node.names:
|
||||
assert not alias.name.startswith("app"), f"illegal import app: {alias.name}"
|
||||
elif isinstance(node, ast.ImportFrom):
|
||||
assert not (node.module or "").startswith("app"), f"illegal from app: {node.module}"
|
||||
|
||||
|
||||
def test_deactivate_stale_wildcard_prefix() -> None:
|
||||
"""deactivate_stale_{avito,yandex,cian} → одна wildcard-запись (как боевой startswith)."""
|
||||
registry, _ = _build_recording_registry()
|
||||
h_avito = resolve_handler("deactivate_stale_avito", registry)
|
||||
h_yandex = resolve_handler("deactivate_stale_yandex", registry)
|
||||
h_cian = resolve_handler("deactivate_stale_cian", registry)
|
||||
assert h_avito is not None
|
||||
assert h_avito is h_yandex is h_cian # один и тот же handler на всё семейство
|
||||
|
||||
|
||||
# ── 2. _claim_run advisory-lock parity ───────────────────────────────────────
|
||||
|
||||
|
||||
class _FakeResult:
|
||||
def __init__(self, *, scalar: Any = None, fetchone: Any = None, fetchall: Any = None) -> None:
|
||||
self._scalar = scalar
|
||||
self._fetchone = fetchone
|
||||
self._fetchall = fetchall or []
|
||||
|
||||
def scalar(self) -> Any:
|
||||
return self._scalar
|
||||
|
||||
def fetchone(self) -> Any:
|
||||
return self._fetchone
|
||||
|
||||
def fetchall(self) -> Any:
|
||||
return self._fetchall
|
||||
|
||||
|
||||
class _FakeClaimDB:
|
||||
"""Мок Session для _claim_run: сценарий has_running_run + advisory-lock."""
|
||||
|
||||
def __init__(self, *, running_states: list[bool], lock: bool = True) -> None:
|
||||
# running_states: очередь ответов has_running_run (по вызовам SELECT 1 FROM scrape_runs)
|
||||
self._running = list(running_states)
|
||||
self._lock = lock
|
||||
self.committed = False
|
||||
self.rolled_back = False
|
||||
self.update_calls = 0
|
||||
|
||||
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
|
||||
sql = str(stmt)
|
||||
if "SELECT 1 FROM scrape_runs" in sql:
|
||||
is_running = self._running.pop(0)
|
||||
return _FakeResult(fetchone=(1,) if is_running else None)
|
||||
if "pg_try_advisory_xact_lock" in sql:
|
||||
return _FakeResult(scalar=self._lock)
|
||||
if "UPDATE scrape_schedules" in sql:
|
||||
self.update_calls += 1
|
||||
return _FakeResult()
|
||||
return _FakeResult()
|
||||
|
||||
def commit(self) -> None:
|
||||
self.committed = True
|
||||
|
||||
def rollback(self) -> None:
|
||||
self.rolled_back = True
|
||||
|
||||
|
||||
def _ctx_with_runs(create_run_ret: int = 42) -> SchedulerContext:
|
||||
runs = MagicMock()
|
||||
runs.create_run = MagicMock(return_value=create_run_ret)
|
||||
return SchedulerContext(
|
||||
config=MagicMock(),
|
||||
matcher=MagicMock(),
|
||||
enrichment=MagicMock(),
|
||||
session_factory=MagicMock(),
|
||||
runs=runs,
|
||||
)
|
||||
|
||||
|
||||
def test_claim_run_happy_path() -> None:
|
||||
db = _FakeClaimDB(running_states=[False, False], lock=True)
|
||||
ctx = _ctx_with_runs(create_run_ret=42)
|
||||
run_id = _claim_run(db, _make_sched("avito_city_sweep"), ctx)
|
||||
assert run_id == 42
|
||||
ctx.runs.create_run.assert_called_once()
|
||||
assert db.update_calls == 1
|
||||
assert db.committed is True
|
||||
|
||||
|
||||
def test_claim_run_skip_already_running() -> None:
|
||||
db = _FakeClaimDB(running_states=[True], lock=True)
|
||||
ctx = _ctx_with_runs()
|
||||
run_id = _claim_run(db, _make_sched("avito_city_sweep"), ctx)
|
||||
assert run_id is None
|
||||
ctx.runs.create_run.assert_not_called()
|
||||
|
||||
|
||||
def test_claim_run_skip_lock_busy() -> None:
|
||||
# pre-check clean, но advisory-lock занят конкурентным тиком → skip
|
||||
db = _FakeClaimDB(running_states=[False], lock=False)
|
||||
ctx = _ctx_with_runs()
|
||||
run_id = _claim_run(db, _make_sched("avito_city_sweep"), ctx)
|
||||
assert run_id is None
|
||||
ctx.runs.create_run.assert_not_called()
|
||||
|
||||
|
||||
def test_claim_run_running_appeared_under_lock() -> None:
|
||||
# pre-check clean, lock взят, но под локом появился running → rollback + skip (double-check)
|
||||
db = _FakeClaimDB(running_states=[False, True], lock=True)
|
||||
ctx = _ctx_with_runs()
|
||||
run_id = _claim_run(db, _make_sched("avito_city_sweep"), ctx)
|
||||
assert run_id is None
|
||||
assert db.rolled_back is True
|
||||
ctx.runs.create_run.assert_not_called()
|
||||
|
||||
|
||||
# ── 3. reap_zombies ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_reap_zombies_counts_and_commits() -> None:
|
||||
db = MagicMock()
|
||||
db.execute.return_value = _FakeResult(fetchall=[MagicMock(id=1), MagicMock(id=2)])
|
||||
n = reap_zombies(db)
|
||||
assert n == 2
|
||||
db.commit.assert_called_once()
|
||||
# порог 6 часов передан в SQL-параметр
|
||||
_stmt, params = db.execute.call_args[0]
|
||||
assert params["interval"] == "6 hours"
|
||||
|
||||
|
||||
def test_reap_zombies_threshold_constant() -> None:
|
||||
assert kit_sched.ZOMBIE_THRESHOLD_HOURS == old_sched.ZOMBIE_THRESHOLD_HOURS == 6
|
||||
|
||||
|
||||
# ── 4. proxy_healthcheck sub-hourly post_claim ───────────────────────────────
|
||||
|
||||
|
||||
def test_reschedule_after_minutes_updates_next_run() -> None:
|
||||
db = MagicMock()
|
||||
hook = reschedule_after_minutes(param="interval_minutes", default=30)
|
||||
ctx = _ctx_with_runs()
|
||||
hook(db, run_id=7, params={"interval_minutes": 15}, ctx=ctx)
|
||||
db.commit.assert_called_once()
|
||||
_stmt, params = db.execute.call_args[0]
|
||||
assert params["mins"] == 15
|
||||
assert params["rid"] == 7
|
||||
|
||||
|
||||
def test_reschedule_after_minutes_default() -> None:
|
||||
db = MagicMock()
|
||||
hook = reschedule_after_minutes()
|
||||
hook(db, run_id=9, params={}, ctx=_ctx_with_runs())
|
||||
_stmt, params = db.execute.call_args[0]
|
||||
assert params["mins"] == 30
|
||||
|
||||
|
||||
# ── 5. _dispatch end-to-end (claim → fresh session → job → close) ────────────
|
||||
|
||||
|
||||
async def test_dispatch_kit_native_sweep_invokes_pipeline() -> None:
|
||||
"""kit-native handler: _dispatch клеймит, открывает сессию и зовёт pipeline.run_*."""
|
||||
run_db = MagicMock()
|
||||
ctx = SchedulerContext(
|
||||
config=MagicMock(),
|
||||
matcher=MagicMock(),
|
||||
enrichment=MagicMock(),
|
||||
session_factory=MagicMock(return_value=run_db),
|
||||
runs=MagicMock(),
|
||||
)
|
||||
ctx.runs.create_run = MagicMock(return_value=100)
|
||||
|
||||
db = _FakeClaimDB(running_states=[False, False], lock=True)
|
||||
registry = build_registry()
|
||||
handler = registry["avito_city_sweep"]
|
||||
|
||||
with patch.object(kit_sched, "run_avito_city_sweep", AsyncMock()) as mock_run:
|
||||
run_id = await _dispatch(handler, db, _make_sched("avito_city_sweep"), ctx)
|
||||
assert run_id == 100
|
||||
# дождаться detached run-задачи
|
||||
await asyncio.gather(*list(ctx._inflight_tasks))
|
||||
|
||||
mock_run.assert_awaited_once()
|
||||
_args, kwargs = mock_run.call_args
|
||||
assert kwargs["run_id"] == 100
|
||||
assert kwargs["config"] is ctx.config
|
||||
assert kwargs["matcher"] is ctx.matcher
|
||||
run_db.close.assert_called_once()
|
||||
|
||||
|
||||
async def test_dispatch_product_handler_invokes_job() -> None:
|
||||
"""Продуктовый handler: _dispatch клеймит и зовёт инжектированный job на свежей сессии."""
|
||||
run_db = MagicMock()
|
||||
ctx = SchedulerContext(
|
||||
config=MagicMock(),
|
||||
matcher=MagicMock(),
|
||||
enrichment=MagicMock(),
|
||||
session_factory=MagicMock(return_value=run_db),
|
||||
runs=MagicMock(),
|
||||
)
|
||||
ctx.runs.create_run = MagicMock(return_value=200)
|
||||
|
||||
db = _FakeClaimDB(running_states=[False, False], lock=True)
|
||||
registry, fired = _build_recording_registry()
|
||||
handler = resolve_handler("sber_index_pull", registry)
|
||||
|
||||
run_id = await _dispatch(handler, db, _make_sched("sber_index_pull"), ctx)
|
||||
assert run_id == 200
|
||||
await asyncio.gather(*list(ctx._inflight_tasks))
|
||||
|
||||
fired["sber_index_pull"].assert_called_once()
|
||||
called_args = fired["sber_index_pull"].call_args[0]
|
||||
assert called_args[0] is run_db # свежая сессия
|
||||
assert called_args[1] == 200 # run_id
|
||||
run_db.close.assert_called_once()
|
||||
|
||||
|
||||
async def test_dispatch_pre_claim_gate_skips() -> None:
|
||||
"""pre_claim → False: _dispatch пропускает claim (cian cookie-gate семантика)."""
|
||||
ctx = _ctx_with_runs()
|
||||
|
||||
async def _deny(db: Any, sch: dict[str, Any], c: Any) -> bool:
|
||||
return False
|
||||
|
||||
job = AsyncMock()
|
||||
|
||||
async def _job(db: Any, run_id: int, params: dict[str, Any], c: Any) -> None:
|
||||
job()
|
||||
|
||||
handler = Handler(_job, "cian_history_backfill", pre_claim=_deny)
|
||||
db = _FakeClaimDB(running_states=[], lock=True)
|
||||
run_id = await _dispatch(handler, db, _make_sched("cian_history_backfill"), ctx)
|
||||
assert run_id is None
|
||||
ctx.runs.create_run.assert_not_called()
|
||||
job.assert_not_called()
|
||||
|
||||
|
||||
async def test_dispatch_post_claim_hook_runs() -> None:
|
||||
"""post_claim вызывается сразу после claim (proxy_healthcheck sub-hourly reschedule)."""
|
||||
run_db = MagicMock()
|
||||
ctx = SchedulerContext(
|
||||
config=MagicMock(),
|
||||
matcher=MagicMock(),
|
||||
enrichment=MagicMock(),
|
||||
session_factory=MagicMock(return_value=run_db),
|
||||
runs=MagicMock(),
|
||||
)
|
||||
ctx.runs.create_run = MagicMock(return_value=300)
|
||||
post = MagicMock()
|
||||
|
||||
async def _job(db: Any, run_id: int, params: dict[str, Any], c: Any) -> None:
|
||||
pass
|
||||
|
||||
handler = Handler(_job, "proxy_healthcheck", post_claim=post)
|
||||
db = _FakeClaimDB(running_states=[False, False], lock=True)
|
||||
run_id = await _dispatch(handler, db, _make_sched("proxy_healthcheck"), ctx)
|
||||
assert run_id == 300
|
||||
await asyncio.gather(*list(ctx._inflight_tasks))
|
||||
post.assert_called_once()
|
||||
assert post.call_args[0][1] == 300 # run_id
|
||||
|
|
@ -0,0 +1,694 @@
|
|||
"""In-app scheduler — strangler-копия `app.services.scheduler` в scraper_kit (#2136).
|
||||
|
||||
Разница с боевым `app.services.scheduler`:
|
||||
|
||||
1. Полная развязка от `app.*` — все продуктовые зависимости инжектируются через
|
||||
`SchedulerContext` (config/matcher/enrichment/session_factory/runs/shutdown),
|
||||
описанные Protocol'ами в `scraper_kit.contracts`. `grep "from app"` — пусто.
|
||||
|
||||
2. **Registry-dispatch вместо copy-paste.** Боевой модуль держит 25+ почти
|
||||
одинаковых `trigger_*_run` (claim → new-session → create_task → close) плюс
|
||||
27-веточный `if/elif` в `scheduler_loop`. Здесь — единственный параметризованный
|
||||
путь `_dispatch()` + `HANDLERS: dict[str, Handler]`. Каждый source — одна запись
|
||||
реестра; общий claim/session/drain-boilerplate вынесен в `_dispatch`.
|
||||
|
||||
Kit-native (sweep-оркестраторы, уже перенесённые в `scraper_kit.orchestration.pipeline`)
|
||||
регистрируются встроенно (`_default_kit_handlers`). Продуктовые джобы, тело которых
|
||||
осталось в `app` (rosreestr_dkp / sber_index / deactivate_stale / *_backfill / …),
|
||||
инжектируются извне как `Handler` через `build_registry(product_handlers=...)`.
|
||||
|
||||
Боевой рантайм (scraper-контейнер) по-прежнему крутит старый `app.services.scheduler` —
|
||||
это COPY, не MOVE. Переключение — отдельный поздний strangler-шаг.
|
||||
|
||||
Критичная concurrency-логика (`_claim_run` advisory-lock + double-check, `reap_zombies`
|
||||
порог, heartbeat, SIGTERM-drain) перенесена ДОСЛОВНО — тот же SQL, то же ветвление.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import random
|
||||
from collections.abc import Callable, Coroutine, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import UTC, datetime, time, timedelta
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from scraper_kit.orchestration import runs as _kit_runs
|
||||
from scraper_kit.orchestration.pipeline import (
|
||||
run_avito_city_sweep,
|
||||
run_avito_full_load,
|
||||
run_avito_newbuilding_sweep,
|
||||
run_cian_city_sweep,
|
||||
run_cian_full_load,
|
||||
run_domclick_city_sweep,
|
||||
run_yandex_city_sweep,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from scraper_kit.contracts import (
|
||||
EnrichmentJobs,
|
||||
HouseMatcher,
|
||||
ScraperConfig,
|
||||
SessionFactory,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Loop interval — check каждую минуту
|
||||
SCHEDULER_TICK_SEC = 60
|
||||
ZOMBIE_THRESHOLD_HOURS = 6
|
||||
|
||||
# Бюджет дренажа детей < scheduler_main._DRAIN_TIMEOUT_S (100s) < docker grace (120s)
|
||||
# (см. #1182 P2 — идентично боевому scheduler'у).
|
||||
_CHILD_DRAIN_TIMEOUT_S = 80.0
|
||||
|
||||
# ── типы job/handler ─────────────────────────────────────────────────────────
|
||||
# Job получает свежую сессию (открыта `_dispatch`), run_id, params и весь контекст
|
||||
# (config/matcher/enrichment/runs) — чтобы иметь доступ к инжектированным зависимостям.
|
||||
Job = Callable[["Session", int, dict[str, Any], "SchedulerContext"], Coroutine[Any, Any, None]]
|
||||
# Pre-claim gate: вернуть False → пропустить claim (напр. cian: нет/протухли cookies).
|
||||
PreClaim = Callable[["Session", dict[str, Any], "SchedulerContext"], Coroutine[Any, Any, bool]]
|
||||
# Post-claim hook: сразу после успешного claim (напр. proxy_healthcheck sub-hourly reschedule).
|
||||
PostClaim = Callable[["Session", int, dict[str, Any], "SchedulerContext"], None]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Handler:
|
||||
"""Одна запись реестра source→обработчик.
|
||||
|
||||
Инкапсулирует всё, что раньше размазывалось по индивидуальному `trigger_*_run`:
|
||||
- `job` — асинхронное тело (SERP/backfill/enrichment). Общий boilerplate
|
||||
(claim, свежая сессия, create_task, close, log) делает `_dispatch`.
|
||||
- `pre_claim` — опциональный гейт ДО claim (cookie-проверка cian → defer).
|
||||
- `post_claim` — опциональный хук сразу ПОСЛЕ claim (sub-hourly reschedule).
|
||||
- `log_name` — имя источника в логах ("scheduler: triggered <log_name> run_id=…").
|
||||
"""
|
||||
|
||||
job: Job
|
||||
log_name: str
|
||||
pre_claim: PreClaim | None = None
|
||||
post_claim: PostClaim | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchedulerContext:
|
||||
"""Инжектируемые зависимости + рантайм-состояние планировщика.
|
||||
|
||||
Заменяет прямые импорты `app.core.config.settings` / `app.services.matching` /
|
||||
`app.services.house_imv_backfill` / `app.core.db.SessionLocal` /
|
||||
`app.services.scrape_runs` / `app.core.shutdown.shutdown_requested`.
|
||||
"""
|
||||
|
||||
config: ScraperConfig
|
||||
matcher: HouseMatcher
|
||||
enrichment: EnrichmentJobs
|
||||
session_factory: SessionFactory
|
||||
shutdown_requested: Callable[[], bool] = lambda: False
|
||||
# runs-модуль (create_run/update_heartbeat/mark_done/… ). По умолчанию — kit-копия
|
||||
# scraper_kit.orchestration.runs; в тестах подменяется recorder'ом.
|
||||
runs: Any = _kit_runs
|
||||
# #1182 P2: strong-ref'ы detached run-задач для graceful SIGTERM-drain'а.
|
||||
_inflight_tasks: set[asyncio.Task[None]] = field(default_factory=set)
|
||||
|
||||
def spawn_tracked(self, coro: Coroutine[Any, Any, None]) -> asyncio.Task[None]:
|
||||
"""create_task + регистрация в _inflight_tasks для graceful-drain'а (#1182 P2).
|
||||
|
||||
strong-ref в set'е держит задачу до завершения (RUF006) и даёт scheduler_loop'у
|
||||
дождаться её на SIGTERM-drain'е; done-callback ретривит exception и убирает задачу.
|
||||
"""
|
||||
task = asyncio.create_task(coro)
|
||||
self._inflight_tasks.add(task)
|
||||
|
||||
def _on_done(t: asyncio.Task[None]) -> None:
|
||||
self._inflight_tasks.discard(t)
|
||||
if not t.cancelled():
|
||||
t.exception()
|
||||
|
||||
task.add_done_callback(_on_done)
|
||||
return task
|
||||
|
||||
async def drain_inflight(self) -> None:
|
||||
"""Дождаться завершения detached run-задач перед teardown'ом (#1182 P2).
|
||||
|
||||
Кооперативные дети дочекивают текущий unit + mark_done и резолвятся;
|
||||
некооперативные упираются в _CHILD_DRAIN_TIMEOUT_S и остаются на внешний
|
||||
hard-cancel fallback. Не busy-spin — один asyncio.wait.
|
||||
"""
|
||||
pending = [t for t in self._inflight_tasks if not t.done()]
|
||||
if not pending:
|
||||
return
|
||||
logger.info("scheduler: draining %d in-flight run task(s) on shutdown", len(pending))
|
||||
_done, still = await asyncio.wait(pending, timeout=_CHILD_DRAIN_TIMEOUT_S)
|
||||
if still:
|
||||
logger.warning(
|
||||
"scheduler: %d task(s) did not drain in %.0fs — leaving for hard-cancel",
|
||||
len(still),
|
||||
_CHILD_DRAIN_TIMEOUT_S,
|
||||
)
|
||||
else:
|
||||
logger.info("scheduler: all in-flight run task(s) drained cleanly")
|
||||
|
||||
|
||||
def compute_next_run_at(
|
||||
window_start_hour: int,
|
||||
window_end_hour: int,
|
||||
*,
|
||||
now: datetime | None = None,
|
||||
interval_days: int = 1,
|
||||
) -> datetime:
|
||||
"""Pick random datetime в window [start, end) UTC, через interval_days суток после now.
|
||||
|
||||
interval_days задаёт каденс источника: 1 (default) = daily (back-compat), 7 = weekly.
|
||||
Берётся из schedule.default_params["interval_days"] вызывающим кодом (_claim_run /
|
||||
_defer_next_run_at); отсутствие ключа → 1 → прежнее ежедневное поведение.
|
||||
|
||||
Если window_end_hour <= window_start_hour → cross-midnight window
|
||||
(например 22→3 → окно 22:00-23:59 ИЛИ 00:00-02:59).
|
||||
"""
|
||||
now = now or datetime.now(tz=UTC)
|
||||
interval_days = max(1, int(interval_days))
|
||||
# Целевая дата = now + interval_days суток (interval_days=1 → завтра, как раньше).
|
||||
target = (now + timedelta(days=interval_days)).date()
|
||||
|
||||
if window_end_hour > window_start_hour:
|
||||
# Обычное окно (например 2..5 → 02:00-04:59)
|
||||
start_seconds = window_start_hour * 3600
|
||||
end_seconds = window_end_hour * 3600
|
||||
rand_seconds = random.randint(start_seconds, end_seconds - 1)
|
||||
return datetime.combine(target, time(0, 0), tzinfo=UTC) + timedelta(seconds=rand_seconds)
|
||||
else:
|
||||
# Cross-midnight (22..3 → 22:00-23:59 + 00:00-02:59)
|
||||
# Длина окна = (24-start) + end часов
|
||||
total_seconds = ((24 - window_start_hour) + window_end_hour) * 3600
|
||||
rand_seconds = random.randint(0, total_seconds - 1)
|
||||
# Если rand попадает в первую часть (start..24)
|
||||
first_half = (24 - window_start_hour) * 3600
|
||||
if rand_seconds < first_half:
|
||||
# interval_days=1: текущая дата (если окно ещё не наступило сегодня) или next day.
|
||||
# interval_days>1: всегда целевая дата (стаггер на N суток вперёд).
|
||||
today_ok = interval_days == 1 and now.hour < window_start_hour
|
||||
base_date = now.date() if today_ok else target
|
||||
return datetime.combine(base_date, time(0, 0), tzinfo=UTC) + timedelta(
|
||||
seconds=window_start_hour * 3600 + rand_seconds
|
||||
)
|
||||
else:
|
||||
# Во второй части (0..end), целевого дня
|
||||
offset = rand_seconds - first_half
|
||||
return datetime.combine(target, time(0, 0), tzinfo=UTC) + timedelta(seconds=offset)
|
||||
|
||||
|
||||
def has_running_run(db: Session, source: str) -> bool:
|
||||
"""Есть ли активный run для source (status='running')."""
|
||||
row = db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT 1 FROM scrape_runs
|
||||
WHERE source = :source AND status = 'running'
|
||||
LIMIT 1
|
||||
"""
|
||||
),
|
||||
{"source": source},
|
||||
).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def reap_zombies(db: Session) -> int:
|
||||
"""Mark scrape_runs as 'zombie' если heartbeat не обновлялся > ZOMBIE_THRESHOLD_HOURS hours."""
|
||||
zombie_interval = f"{ZOMBIE_THRESHOLD_HOURS} hours"
|
||||
result = db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE scrape_runs
|
||||
SET status = 'zombie', finished_at = NOW()
|
||||
WHERE status = 'running'
|
||||
AND (heartbeat_at IS NULL
|
||||
OR heartbeat_at < NOW() - CAST(:interval AS interval))
|
||||
RETURNING id
|
||||
"""
|
||||
),
|
||||
{"interval": zombie_interval},
|
||||
)
|
||||
rows = result.fetchall()
|
||||
db.commit()
|
||||
if rows:
|
||||
logger.warning("scheduler: reaped %d zombie runs: %s", len(rows), [r.id for r in rows])
|
||||
return len(rows)
|
||||
|
||||
|
||||
def _claim_run(db: Session, schedule_row: dict[str, Any], ctx: SchedulerContext) -> int | None:
|
||||
"""Claim run: INSERT scrape_runs + UPDATE scrape_schedules next_run_at.
|
||||
|
||||
Returns run_id или None если уже есть running run для этого source.
|
||||
Общий helper для всех source-handler'ов. Advisory-lock concurrency-логика перенесена
|
||||
ДОСЛОВНО из боевого scheduler'а (создание run — через инжектированный ctx.runs).
|
||||
"""
|
||||
source = schedule_row["source"]
|
||||
if has_running_run(db, source):
|
||||
logger.info("scheduler: skip — already running for source=%s", source)
|
||||
return None
|
||||
|
||||
# #750: атомарный claim через transaction-scoped advisory lock. Сериализует
|
||||
# конкурентные _claim_run одного source (redeploy-overlap старого+нового контейнера
|
||||
# / 2 реплики), иначе оба пройдут has_running_run-pre-check и запустят ДВОЙНОЙ sweep
|
||||
# → 2× rate к Avito/Cian → бан. pg_try_advisory_xact_lock НЕ блокирует: lock занят
|
||||
# другим тиком → false → return None. Лок авто-снимается на commit/rollback этой
|
||||
# транзакции (has_running_run выше — быстрый pre-check, обычно без взятия лока).
|
||||
got_lock = db.execute(
|
||||
text("SELECT pg_try_advisory_xact_lock(hashtext(:source))"),
|
||||
{"source": source},
|
||||
).scalar()
|
||||
if not got_lock:
|
||||
logger.info("scheduler: skip — concurrent claim in progress for source=%s", source)
|
||||
return None
|
||||
|
||||
# Double-checked под локом: конкурентный тик мог закоммитить running-run МЕЖДУ
|
||||
# pre-check и взятием лока (затем освободить lock на commit). Перепроверяем под
|
||||
# локом — иначе словим двойной run на узком окне.
|
||||
if has_running_run(db, source):
|
||||
logger.info("scheduler: skip — running appeared under lock for source=%s", source)
|
||||
db.rollback() # освобождаем advisory lock (claim не состоялся)
|
||||
return None
|
||||
|
||||
params = schedule_row.get("default_params") or {}
|
||||
run_id = ctx.runs.create_run(db, source=source, params=params)
|
||||
|
||||
next_at = compute_next_run_at(
|
||||
schedule_row["window_start_hour"],
|
||||
schedule_row["window_end_hour"],
|
||||
interval_days=int(params.get("interval_days", 1)),
|
||||
)
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE scrape_schedules
|
||||
SET last_run_id = :run_id, last_run_at = NOW(),
|
||||
next_run_at = :next_at, updated_at = NOW()
|
||||
WHERE source = :source
|
||||
"""
|
||||
),
|
||||
{"run_id": run_id, "next_at": next_at, "source": source},
|
||||
)
|
||||
db.commit()
|
||||
logger.info(
|
||||
"scheduler: claimed run_id=%d source=%s next_run_at=%s",
|
||||
run_id,
|
||||
source,
|
||||
next_at.isoformat(),
|
||||
)
|
||||
return run_id
|
||||
|
||||
|
||||
def _defer_next_run_at(db: Session, schedule_row: dict[str, Any]) -> None:
|
||||
"""Сдвинуть next_run_at на следующее окно БЕЗ создания run (#1522).
|
||||
|
||||
Используется когда pre_claim делает early-return (например cian: отсутствуют/протухли
|
||||
cookies) ДО _claim_run. Без этого get_due_schedules переотбирает schedule на каждом
|
||||
тике (SCHEDULER_TICK_SEC), и pre-check гоняется раз в минуту круглосуточно.
|
||||
"""
|
||||
source = schedule_row["source"]
|
||||
params = schedule_row.get("default_params") or {}
|
||||
next_at = compute_next_run_at(
|
||||
schedule_row["window_start_hour"],
|
||||
schedule_row["window_end_hour"],
|
||||
interval_days=int(params.get("interval_days", 1)),
|
||||
)
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE scrape_schedules
|
||||
SET next_run_at = :next_at, updated_at = NOW()
|
||||
WHERE source = :source
|
||||
"""
|
||||
),
|
||||
{"next_at": next_at, "source": source},
|
||||
)
|
||||
db.commit()
|
||||
logger.info(
|
||||
"scheduler: deferred next_run_at source=%s next_run_at=%s (no run claimed)",
|
||||
source,
|
||||
next_at.isoformat(),
|
||||
)
|
||||
|
||||
|
||||
def reschedule_after_minutes(*, param: str = "interval_minutes", default: int = 30) -> PostClaim:
|
||||
"""Фабрика post_claim-хука: sub-hourly re-schedule сразу после claim (#2162).
|
||||
|
||||
Боевой scheduler держит суточную гранулярность в compute_next_run_at, а
|
||||
proxy_healthcheck хочет каденс раз в N минут. Хук СРАЗУ после claim переопределяет
|
||||
next_run_at на now() + interval_minutes (source-специфично, не трогая shared
|
||||
compute_next_run_at). Если run ещё идёт на следующем тике — has_running_run в
|
||||
_claim_run вернёт None (skip) и next_run_at не сбросится → авто-throttle по факту.
|
||||
"""
|
||||
|
||||
def _hook(db: Session, run_id: int, params: dict[str, Any], ctx: SchedulerContext) -> None:
|
||||
# source читаем из ЗАКЛЕЙМЛЕННОГО schedule — берём из params-owner через closure нельзя,
|
||||
# поэтому апдейтим по run_id → source (schedule owner). Проще: UPDATE по source из
|
||||
# scrape_runs текущего run_id, что эквивалентно WHERE source = <this source>.
|
||||
minutes = int(params.get(param, default))
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE scrape_schedules
|
||||
SET next_run_at = now() + make_interval(mins => CAST(:mins AS integer)),
|
||||
updated_at = now()
|
||||
WHERE source = (SELECT source FROM scrape_runs WHERE id = CAST(:rid AS bigint))
|
||||
"""
|
||||
),
|
||||
{"mins": minutes, "rid": run_id},
|
||||
)
|
||||
db.commit()
|
||||
|
||||
return _hook
|
||||
|
||||
|
||||
# ── единственный параметризованный dispatch (заменяет 25 trigger_* + 27 if/elif) ──
|
||||
|
||||
|
||||
async def _dispatch(
|
||||
handler: Handler,
|
||||
db: Session,
|
||||
schedule_row: dict[str, Any],
|
||||
ctx: SchedulerContext,
|
||||
) -> int | None:
|
||||
"""Общий путь claim→spawn для ЛЮБОГО source (весь boilerplate прежних trigger_*_run).
|
||||
|
||||
1. Опциональный pre_claim-гейт (False → skip без claim).
|
||||
2. _claim_run (advisory-lock; None → уже running / concurrent claim).
|
||||
3. Опциональный post_claim-хук (sub-hourly reschedule).
|
||||
4. spawn detached run-задачи: свежая сессия → handler.job → close (+ exception-log).
|
||||
"""
|
||||
if handler.pre_claim is not None:
|
||||
proceed = await handler.pre_claim(db, schedule_row, ctx)
|
||||
if not proceed:
|
||||
return None
|
||||
|
||||
run_id = _claim_run(db, schedule_row, ctx)
|
||||
if run_id is None:
|
||||
return None
|
||||
|
||||
params = schedule_row.get("default_params") or {}
|
||||
|
||||
if handler.post_claim is not None:
|
||||
handler.post_claim(db, run_id, params, ctx)
|
||||
|
||||
async def _run() -> None:
|
||||
run_db = ctx.session_factory()
|
||||
try:
|
||||
await handler.job(run_db, run_id, params, ctx)
|
||||
except Exception:
|
||||
logger.exception("scheduler: %s crashed run_id=%d", handler.log_name, run_id)
|
||||
finally:
|
||||
run_db.close()
|
||||
|
||||
ctx.spawn_tracked(_run())
|
||||
logger.info("scheduler: triggered %s run_id=%d", handler.log_name, run_id)
|
||||
return run_id
|
||||
|
||||
|
||||
def resolve_handler(source: str, registry: Mapping[str, Handler]) -> Handler | None:
|
||||
"""Найти handler по source: exact-match, затем wildcard-префикс ("deactivate_stale_*").
|
||||
|
||||
Единственная точка ветвления вместо 27 if/elif. Wildcard-ключ (кончается на "*")
|
||||
матчит по префиксу — покрывает семейство deactivate_stale_avito/yandex/cian одной
|
||||
записью реестра (как боевой `source.startswith("deactivate_stale_")`).
|
||||
"""
|
||||
handler = registry.get(source)
|
||||
if handler is not None:
|
||||
return handler
|
||||
for key, h in registry.items():
|
||||
if key.endswith("*") and source.startswith(key[:-1]):
|
||||
return h
|
||||
return None
|
||||
|
||||
|
||||
# ── kit-native sweep-обработчики (тело в scraper_kit.orchestration.pipeline) ──────
|
||||
# Param-чтение идентично боевым trigger_*_run — та же семантика окон/дефолтов.
|
||||
|
||||
|
||||
async def _job_avito_city_sweep(
|
||||
db: Session, run_id: int, params: dict[str, Any], ctx: SchedulerContext
|
||||
) -> None:
|
||||
await run_avito_city_sweep(
|
||||
db,
|
||||
run_id=run_id,
|
||||
config=ctx.config,
|
||||
matcher=ctx.matcher,
|
||||
enrichment=ctx.enrichment,
|
||||
shutdown_requested=ctx.shutdown_requested,
|
||||
pages_per_anchor=int(params.get("pages_per_anchor", 3)),
|
||||
detail_top_n=int(params.get("detail_top_n", 20)),
|
||||
request_delay_sec=float(params.get("request_delay_sec", 7.0)),
|
||||
enrich_houses=bool(params.get("enrich_houses", True)),
|
||||
radius_m=int(params.get("radius_m", 1500)),
|
||||
)
|
||||
|
||||
|
||||
async def _job_avito_newbuilding_sweep(
|
||||
db: Session, run_id: int, params: dict[str, Any], ctx: SchedulerContext
|
||||
) -> None:
|
||||
await run_avito_newbuilding_sweep(
|
||||
db,
|
||||
run_id=run_id,
|
||||
config=ctx.config,
|
||||
matcher=ctx.matcher,
|
||||
shutdown_requested=ctx.shutdown_requested,
|
||||
pages=int(params.get("pages", 20)),
|
||||
request_delay_sec=float(params.get("request_delay_sec", 7.0)),
|
||||
)
|
||||
|
||||
|
||||
async def _job_avito_full_load(
|
||||
db: Session, run_id: int, params: dict[str, Any], ctx: SchedulerContext
|
||||
) -> None:
|
||||
_incremental_days = params.get("incremental_days")
|
||||
incremental_days = int(_incremental_days) if _incremental_days is not None else None
|
||||
await run_avito_full_load(
|
||||
db,
|
||||
run_id=run_id,
|
||||
config=ctx.config,
|
||||
matcher=ctx.matcher,
|
||||
shutdown_requested=ctx.shutdown_requested,
|
||||
price_cap_per_bucket=int(params.get("price_cap_per_bucket", 1400)),
|
||||
concurrency=int(params.get("concurrency", 5)),
|
||||
request_delay_sec=float(params.get("request_delay_sec", 7.0)),
|
||||
secondary_only=bool(params.get("secondary_only", True)),
|
||||
resume_run_id=None,
|
||||
incremental_days=incremental_days,
|
||||
)
|
||||
|
||||
|
||||
async def _job_avito_full_load_exhaustive(
|
||||
db: Session, run_id: int, params: dict[str, Any], ctx: SchedulerContext
|
||||
) -> None:
|
||||
# incremental_days форсится в None → полный exhaustive обход (weekly cadence).
|
||||
await run_avito_full_load(
|
||||
db,
|
||||
run_id=run_id,
|
||||
config=ctx.config,
|
||||
matcher=ctx.matcher,
|
||||
shutdown_requested=ctx.shutdown_requested,
|
||||
price_cap_per_bucket=int(params.get("price_cap_per_bucket", 1400)),
|
||||
concurrency=int(params.get("concurrency", 5)),
|
||||
request_delay_sec=float(params.get("request_delay_sec", 7.0)),
|
||||
secondary_only=bool(params.get("secondary_only", True)),
|
||||
resume_run_id=None,
|
||||
incremental_days=None,
|
||||
)
|
||||
|
||||
|
||||
async def _job_yandex_city_sweep(
|
||||
db: Session, run_id: int, params: dict[str, Any], ctx: SchedulerContext
|
||||
) -> None:
|
||||
await run_yandex_city_sweep(
|
||||
db,
|
||||
run_id=run_id,
|
||||
config=ctx.config,
|
||||
matcher=ctx.matcher,
|
||||
enrichment=ctx.enrichment,
|
||||
shutdown_requested=ctx.shutdown_requested,
|
||||
pages_per_anchor=int(params.get("pages_per_anchor", 2)),
|
||||
request_delay_sec=float(params.get("request_delay_sec", 9.0)),
|
||||
radius_m=int(params.get("radius_m", 1500)),
|
||||
enrich_address=bool(params.get("enrich_address", True)),
|
||||
segments=params.get("segments"),
|
||||
)
|
||||
|
||||
|
||||
async def _job_cian_city_sweep(
|
||||
db: Session, run_id: int, params: dict[str, Any], ctx: SchedulerContext
|
||||
) -> None:
|
||||
await run_cian_city_sweep(
|
||||
db,
|
||||
run_id=run_id,
|
||||
config=ctx.config,
|
||||
matcher=ctx.matcher,
|
||||
shutdown_requested=ctx.shutdown_requested,
|
||||
pages_per_anchor=int(params.get("pages_per_anchor", 3)),
|
||||
request_delay_sec=float(params.get("request_delay_sec", 5.0)),
|
||||
radius_m=int(params.get("radius_m", 1500)),
|
||||
detail_top_n=int(params.get("detail_top_n", 10)),
|
||||
enrich_houses=bool(params.get("enrich_houses", True)),
|
||||
newbuilding_only=bool(params.get("newbuilding_only", True)),
|
||||
)
|
||||
|
||||
|
||||
async def _job_cian_full_load(
|
||||
db: Session, run_id: int, params: dict[str, Any], ctx: SchedulerContext
|
||||
) -> None:
|
||||
await run_cian_full_load(
|
||||
db,
|
||||
run_id=run_id,
|
||||
config=ctx.config,
|
||||
matcher=ctx.matcher,
|
||||
shutdown_requested=ctx.shutdown_requested,
|
||||
price_cap_per_bucket=int(params.get("price_cap_per_bucket", 1400)),
|
||||
concurrency=int(params.get("concurrency", 5)),
|
||||
request_delay_sec=float(params.get("request_delay_sec", 4.0)),
|
||||
enrich_detail=bool(params.get("enrich_detail", False)),
|
||||
detail_top_n=int(params.get("detail_top_n", 0)),
|
||||
resume_run_id=None,
|
||||
)
|
||||
|
||||
|
||||
async def _job_domclick_city_sweep(
|
||||
db: Session, run_id: int, params: dict[str, Any], ctx: SchedulerContext
|
||||
) -> None:
|
||||
await run_domclick_city_sweep(
|
||||
db,
|
||||
run_id=run_id,
|
||||
config=ctx.config,
|
||||
matcher=ctx.matcher,
|
||||
shutdown_requested=ctx.shutdown_requested,
|
||||
city_id=int(params.get("city_id", 4)),
|
||||
rooms=params.get("rooms"),
|
||||
pages=int(params.get("pages_per_anchor", 5)),
|
||||
request_delay_sec=float(params.get("request_delay_sec", 6.0)),
|
||||
)
|
||||
|
||||
|
||||
def _default_kit_handlers() -> dict[str, Handler]:
|
||||
"""Встроенные kit-native sweep-обработчики (тело в scraper_kit.orchestration.pipeline)."""
|
||||
return {
|
||||
"avito_city_sweep": Handler(_job_avito_city_sweep, "avito_city_sweep"),
|
||||
"avito_newbuilding_sweep": Handler(_job_avito_newbuilding_sweep, "avito_newbuilding_sweep"),
|
||||
"avito_full_load": Handler(_job_avito_full_load, "avito_full_load"),
|
||||
"avito_full_load_exhaustive": Handler(
|
||||
_job_avito_full_load_exhaustive, "avito_full_load_exhaustive"
|
||||
),
|
||||
"yandex_city_sweep": Handler(_job_yandex_city_sweep, "yandex_city_sweep"),
|
||||
"cian_city_sweep": Handler(_job_cian_city_sweep, "cian_city_sweep"),
|
||||
"cian_full_load": Handler(_job_cian_full_load, "cian_full_load"),
|
||||
"domclick_city_sweep": Handler(_job_domclick_city_sweep, "domclick_city_sweep"),
|
||||
}
|
||||
|
||||
|
||||
def build_registry(product_handlers: Mapping[str, Handler] | None = None) -> dict[str, Handler]:
|
||||
"""Собрать реестр source→Handler: kit-native sweeps + инжектированные продуктовые джобы.
|
||||
|
||||
`product_handlers` — джобы, тело которых осталось в `app` (rosreestr_dkp / sber_index /
|
||||
deactivate_stale_* / *_backfill / cadastral_geo_match / house_* / proxy_healthcheck / …).
|
||||
Их строит app-side wiring и передаёт сюда. Продуктовые ключи МОГУТ переопределять
|
||||
kit-native (последнее слово за продуктом).
|
||||
|
||||
Wildcard-ключи ("deactivate_stale_*") резолвятся `resolve_handler` по префиксу.
|
||||
"""
|
||||
registry = dict(_default_kit_handlers())
|
||||
if product_handlers:
|
||||
registry.update(product_handlers)
|
||||
return registry
|
||||
|
||||
|
||||
def get_due_schedules(db: Session) -> list[dict[str, Any]]:
|
||||
"""SELECT scrape_schedules WHERE enabled AND (next_run_at IS NULL OR next_run_at <= NOW())."""
|
||||
rows = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT id, source, enabled, window_start_hour, window_end_hour,
|
||||
default_params, last_run_id, last_run_at, next_run_at
|
||||
FROM scrape_schedules
|
||||
WHERE enabled = true
|
||||
AND (next_run_at IS NULL OR next_run_at <= NOW())
|
||||
"""
|
||||
),
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
async def scheduler_loop(ctx: SchedulerContext, registry: Mapping[str, Handler]) -> None:
|
||||
"""Бесконечный async loop — tick каждые SCHEDULER_TICK_SEC секунд.
|
||||
|
||||
Структура (reap → get_due → dispatch → SIGTERM-drain) идентична боевому
|
||||
scheduler'у; 27-веточный if/elif заменён единственным `resolve_handler` + `_dispatch`.
|
||||
"""
|
||||
logger.info("scheduler: started (tick=%ds)", SCHEDULER_TICK_SEC)
|
||||
# Initial sleep 30s чтобы дать FastAPI startup завершиться
|
||||
await asyncio.sleep(30)
|
||||
while True:
|
||||
# #1182 Phase 2: кооперативный SIGTERM-drain. Не reap'аем и не claim'аем
|
||||
# новые run'ы во время shutdown — даём текущему dispatch'у докатиться и выходим.
|
||||
if ctx.shutdown_requested():
|
||||
logger.info("scheduler: SIGTERM-drain — stop claiming new runs, exiting tick loop")
|
||||
break
|
||||
try:
|
||||
db = ctx.session_factory()
|
||||
try:
|
||||
# Reap zombies first
|
||||
reap_zombies(db)
|
||||
# Process due schedules
|
||||
due = get_due_schedules(db)
|
||||
for sch in due:
|
||||
source = sch["source"]
|
||||
handler = resolve_handler(source, registry)
|
||||
if handler is None:
|
||||
logger.warning("scheduler: unknown source=%s, skip", source)
|
||||
else:
|
||||
await _dispatch(handler, db, sch, ctx)
|
||||
|
||||
# #1182 Phase 2: после каждого dispatch'а проверяем drain — текущий
|
||||
# run уже отпущен в свою asyncio-задачу (сам докоммитит/выйдет по
|
||||
# своему checkpoint'у), а новые в этом тике не запускаем.
|
||||
if ctx.shutdown_requested():
|
||||
logger.info(
|
||||
"scheduler: SIGTERM-drain — stop dispatch after source=%s", source
|
||||
)
|
||||
break
|
||||
finally:
|
||||
db.close()
|
||||
except Exception:
|
||||
logger.exception("scheduler: tick failed")
|
||||
# Промежуточная проверка перед 60s-сном: выходим сразу, не ждём целый тик.
|
||||
if ctx.shutdown_requested():
|
||||
logger.info("scheduler: SIGTERM-drain — exiting tick loop after dispatch")
|
||||
break
|
||||
await asyncio.sleep(SCHEDULER_TICK_SEC)
|
||||
|
||||
# Tick-loop вышел только по SIGTERM-drain'у (иначе while True бесконечен): дожидаемся
|
||||
# detached run-задач (spawn_tracked) — пусть докоммитят текущий unit и сделают
|
||||
# mark_done, а не дадим asyncio.run() teardown'у хард-кансельнуть их mid-await (#1182).
|
||||
await ctx.drain_inflight()
|
||||
logger.info("scheduler: tick loop exited (in-flight drain complete)")
|
||||
|
||||
|
||||
__all__ = [
|
||||
"SCHEDULER_TICK_SEC",
|
||||
"ZOMBIE_THRESHOLD_HOURS",
|
||||
"Handler",
|
||||
"SchedulerContext",
|
||||
"build_registry",
|
||||
"compute_next_run_at",
|
||||
"get_due_schedules",
|
||||
"has_running_run",
|
||||
"reap_zombies",
|
||||
"reschedule_after_minutes",
|
||||
"resolve_handler",
|
||||
"scheduler_loop",
|
||||
]
|
||||
Loading…
Add table
Reference in a new issue