diff --git a/tradein-mvp/backend/app/core/config.py b/tradein-mvp/backend/app/core/config.py index b29c7198..3893fd59 100644 --- a/tradein-mvp/backend/app/core/config.py +++ b/tradein-mvp/backend/app/core/config.py @@ -18,11 +18,12 @@ class Settings(BaseSettings): # Default true preserves existing behaviour. scheduler_enable: bool = Field(default=True, validation_alias="SCHEDULER_ENABLE") - # ── strangler-миграция scheduler → scraper_kit (#2192, ship-dark) ───────── - # Дефолт False → scheduler_main идёт СТАРЫМ путём (app.services.scheduler.scheduler_loop), - # прод не меняет поведение на деплое. True → kit-путь (scraper_kit.orchestration.scheduler - # + product_handlers). Переключается вручную после prod-smoke; боевой scheduler остаётся - # мгновенным fallback. + # ── strangler-миграция scheduler → scraper_kit (#2192, завершена #2397 Part C) ──── + # scheduler_main.py теперь безусловно идёт kit-путём (scraper_kit.orchestration.scheduler + # + product_handlers) — legacy app.services.scheduler.scheduler_loop fallback удалён + # (был мёртвым грузом: прод давно на USE_KIT_SCHEDULER=true). Поле оставлено (extra= + # "ignore" в model_config защищает от startup-краха на leftover env var), но больше + # ни на что не влияет. use_kit_scheduler: bool = Field(default=False, validation_alias="USE_KIT_SCHEDULER") # Proxy-pool флаги — объявлены заранее (потребители P3/P4 отдельно), дефолт False. use_proxy_pool_curl: bool = Field(default=False, validation_alias="USE_PROXY_POOL_CURL") diff --git a/tradein-mvp/backend/app/main.py b/tradein-mvp/backend/app/main.py index 5f7df92d..55a13911 100644 --- a/tradein-mvp/backend/app/main.py +++ b/tradein-mvp/backend/app/main.py @@ -6,7 +6,6 @@ Standalone версия, выделена из основного gendesign repo from __future__ import annotations -import asyncio import logging import os import re @@ -31,7 +30,6 @@ from app.core.db import SessionLocal from app.core.fdw import ensure_fdw_user_mapping from app.core.ratelimit import RateLimitMiddleware from app.observability.sentry_scrub import scrub_pii_event -from app.services.scheduler import scheduler_loop logger = logging.getLogger(__name__) @@ -85,23 +83,11 @@ async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: except Exception: logger.exception("FDW user mapping bootstrap failed — cadastral queries may fail") - # Startup: launch scheduler background task (disabled when SCHEDULER_ENABLE=false, - # e.g. when using the systemd timer trigger — see tradein-mvp/ops/systemd/ #581). - scheduler_task = None - if settings.scheduler_enable: - scheduler_task = asyncio.create_task(scheduler_loop()) - logger.info("FastAPI lifespan: scheduler task spawned") - else: - logger.info("FastAPI lifespan: in-app scheduler disabled (SCHEDULER_ENABLE=false)") + # #2397 Part C: legacy in-app scheduler launch removed — scheduling lives exclusively + # in the tradein-scraper container (`python -m app.scheduler_main`, kit scheduler). + # Prod backend has always run with SCHEDULER_ENABLE=false (see docker-compose.prod.yml); + # this API process never actually launched scheduler_loop() in production. yield - # Shutdown: cancel scheduler if it was started - if scheduler_task is not None: - scheduler_task.cancel() - try: - await scheduler_task - except asyncio.CancelledError: - pass - logger.info("FastAPI lifespan: scheduler cancelled cleanly") app = FastAPI( diff --git a/tradein-mvp/backend/app/scheduler_main.py b/tradein-mvp/backend/app/scheduler_main.py index a9f2e58c..3bb8af4a 100644 --- a/tradein-mvp/backend/app/scheduler_main.py +++ b/tradein-mvp/backend/app/scheduler_main.py @@ -6,8 +6,8 @@ Запуск: python -m app.scheduler_main -Zombie-reap: reap_zombies() вызывается в начале каждого тика scheduler_loop() -(см. app/services/scheduler.py), дублировать здесь не нужно. +Zombie-reap: reap_zombies() вызывается в начале каждого тика kit scheduler_loop +(см. scraper_kit/orchestration/scheduler.py), дублировать здесь не нужно. """ from __future__ import annotations @@ -163,23 +163,19 @@ async def _run_kit_scheduler() -> None: async def _run() -> None: - """Async entrypoint: scheduler_loop() с кооперативным SIGTERM/SIGINT-drain'ом. + """Async entrypoint: kit-scheduler с кооперативным SIGTERM/SIGINT-drain'ом. SIGTERM/SIGINT → request_shutdown() (НЕ task.cancel()): бегущий scrape-unit - докоммитит текущую карточку и выйдет сам на ближайшем between-unit checkpoint'е - (scheduler_loop / avito_detail_backfill / import_rosreestr_dkp). Safety-net в - _await_scheduler гарантирует выход в пределах docker grace. + докоммитит текущую карточку и выйдет сам на ближайшем between-unit checkpoint'е. + Safety-net в _await_scheduler гарантирует выход в пределах docker grace. - Развилка #2192 (ship-dark): USE_KIT_SCHEDULER=false (дефолт) → боевой - app.services.scheduler.scheduler_loop без изменений; =true → kit-путь. + #2397 Part C: legacy app.services.scheduler.scheduler_loop fallback (USE_KIT_SCHEDULER= + false ship-dark путь из #2192) удалён — kit (_run_kit_scheduler) теперь единственный + путь. Прод уже давно на kit (USE_KIT_SCHEDULER=true), legacy был мёртвым грузом. + settings.use_kit_scheduler остаётся в конфиге (extra="ignore" защищает от startup-краха + на leftover env var), но на ветвление больше не влияет. """ - if settings.use_kit_scheduler: - logger.info("scheduler_main: USE_KIT_SCHEDULER=true — running kit-scheduler path") - task = asyncio.create_task(_run_kit_scheduler()) - else: - from app.services.scheduler import scheduler_loop - - task = asyncio.create_task(scheduler_loop()) + task = asyncio.create_task(_run_kit_scheduler()) loop = asyncio.get_running_loop() diff --git a/tradein-mvp/backend/app/services/house_dedup_merge.py b/tradein-mvp/backend/app/services/house_dedup_merge.py index 37ab4ba8..ede3356b 100644 --- a/tradein-mvp/backend/app/services/house_dedup_merge.py +++ b/tradein-mvp/backend/app/services/house_dedup_merge.py @@ -732,9 +732,9 @@ def merge_duplicate_houses(db: Session, *, dry_run: bool = False) -> dict[str, i def run_house_dedup_merge(db: Session, *, run_id: int, params: dict) -> dict[str, int]: """Run-lifecycle wrapper for the recurring house-dedup merge (sync, DB-only). - Launched by the in-app scheduler (source='house_dedup_merge') via - trigger_house_dedup_merge_run, or manually. Mirrors run_cadastral_geo_match: pure - internal DB op, finalises scrape_runs (mark_done / mark_failed) with counters. + Launched by the kit scheduler (source='house_dedup_merge') via + product_handlers._job_house_dedup_merge, or manually. Mirrors run_cadastral_geo_match: + pure internal DB op, finalises scrape_runs (mark_done / mark_failed) with counters. Params (default_params jsonb): dry_run: compute + return counts then ROLLBACK without writing (default false). diff --git a/tradein-mvp/backend/app/services/scheduler.py b/tradein-mvp/backend/app/services/scheduler.py index 8c7eb6e4..96b0ce88 100644 --- a/tradein-mvp/backend/app/services/scheduler.py +++ b/tradein-mvp/backend/app/services/scheduler.py @@ -1,242 +1,42 @@ -"""In-app scheduler — periodically triggers configured scrapes based on scrape_schedules table. +"""Shared scheduler helpers (#2397 Part C — legacy scheduler_loop removed). -Запускается в FastAPI lifespan (app.main lifespan context manager) при startup. -Логика: -1. Каждые 60 sec query scrape_schedules WHERE enabled=true AND next_run_at <= NOW(). -2. Для каждого: проверить нет ли уже running run этого source — если нет, trigger. -3. После trigger — recompute next_run_at через interval_days суток (из default_params, - default 1 = daily; например avito_full_load_exhaustive = 7 → weekly). -4. На old runs (orphaned status='running' > 6h без heartbeat) — пометить как 'zombie'. +История: до #2192 этот модуль нёс полный in-app asyncio scheduler (`scheduler_loop`, +27-веточный `if/elif` dispatch, `trigger_*_run` per-source launchers, `get_due_schedules`, +`reap_zombies`, `_claim_run` advisory-lock claim, `_spawn_tracked`/`_drain_inflight` +graceful-drain machinery). #2192 представил kit-путь (`scraper_kit.orchestration.scheduler` ++ `app.services.product_handlers`) как ship-dark alternative; прод переключился на него +(`USE_KIT_SCHEDULER=true` в tradein-scraper, см. docker-compose.prod.yml). #2397 Part C +убрал legacy `scheduler_loop` и все `trigger_*`/dispatch-функции — kit теперь единственный +scheduling-путь (`app/scheduler_main.py` безусловно запускает `_run_kit_scheduler()`). -Sources: - - avito_city_sweep → run_avito_city_sweep (scrape_pipeline.py) - - avito_full_load → run_avito_full_load (региональный сбор Avito ЕКБ - ВТОРИЧКИ без anchor'ов — room×price адаптивное партиционирование - через pmin/pmax, secondary_only, incremental on_bucket save; - обходит SERP-cap ~5000/запрос. Окно 13:00-15:00 UTC чтобы не - конфликтовать на shared apw-прокси с avito_city_sweep (6-7), - avito_newbuilding (2-5), avito_detail_backfill (9-12). - ПО УМОЛЧАНИЮ инкрементальный: incremental_days из default_params - → shallow обход с date early-stop, избегает page-36 429-банов) - - avito_full_load_exhaustive → run_avito_full_load с incremental_days=None (полный - exhaustive обход; weekly cadence через interval_days=7 в - default_params — освежает last_seen 10-дневный delisting TTL - + ловит тихие правки цены) - - avito_newbuilding_sweep → run_avito_newbuilding_sweep (scrape_pipeline.py; dedicated - novostroyka-filtered citywide SERP → save, 100% new-build cards) - - yandex_city_sweep → run_yandex_city_sweep (scrape_pipeline.py, #561; shipped DORMANT) - - cian_history_backfill → run_cian_backfill (this module, #560) - - rosreestr_dkp_import → import_rosreestr_dkp (this module, #563) - - listing_source_snapshot → snapshot_listing_sources (tasks/listing_source_snapshot.py, #570; - pure internal DB snapshot — no external calls, enabled by default) - - asking_to_sold_ratio_refresh → recompute_asking_to_sold_ratios - (tasks/asking_to_sold_ratio.py, #648 Stage 4; pure internal DB - re-derivation of the asking→sold ratios — no external calls, - enabled by default; window after rosreestr_dkp_import) - - yandex_address_backfill → run_yandex_address_backfill - (tasks/yandex_address_backfill.py, #855; EKB-pilot: fetches - yandex detail pages, extracts full address from , - updates listings.address for street-only addresses; - nightly 02:00-03:00 UTC, before refresh_search_matview) - - deactivate_stale_* → trigger_deactivate_stale_run → deactivate_stale_listings - (tasks/deactivate_stale_avito.py, #759 + segment-aware - generalisation; marks listings with last_seen_at > TTL days as - is_active=false — no DELETE, no external calls. Source prefix - dispatch covers deactivate_stale_avito (all segments, TTL=10) + - deactivate_stale_yandex/cian (vtorichka only, TTL=30, segments - from default_params). Window after the source sweep.) - - sber_index_pull → run_sber_index_pull - (tasks/sber_index_pull.py, #887; monthly pull of СберИндекс - city-level secondary-housing price index — pure HTTP+DB, no - anti-bot/proxy; window 05:00-06:00 UTC, ~28-day cadence) - - rosreestr_quarter_poll → run_rosreestr_quarter_poll - (tasks/rosreestr_quarter_poll.py, #888; monthly HEAD-check of - Rosreestr open-data portal for new quarter availability — pure - HTTP detect+alert, NO download/shell-out; window 07:00-08:00 UTC, - ~28-day cadence) - - newbuilding_enrich → run_newbuilding_enrich - (tasks/newbuilding_enrich_backfill.py, #973; nightly slice of the - cian_newbuilding houses enrichment — resolves cian_zhk_url, fetches - the ЖК page, persists houses_price_dynamics / house_reliability_checks - / house_reviews. Cian HTTP (curl_cffi chrome120) + anti-bot delay. - Idempotent: skips houses already enriched, per-house SAVEPOINT, so - each fire drains the next `limit` pending houses; window 00:00-01:00 - UTC = 03:00-04:00 МСК, before the 01:00+ UTC sweep block) - - yandex_newbuilding_sweep → enrich_yandex_newbuilding_sweep - (tasks/yandex_newbuilding_sweep.py, #974; enrichment ЖК в - market.yandex_jk_enrichment через BrowserFetcher; shipped DORMANT — - enable manually after deploy verified; window 02:00-05:00 UTC) - - cian_city_sweep → run_cian_city_sweep (newbuilding Phase 4, #973; shipped DORMANT — - proxy dead, enable manually after proxy restored. NB-only с #973+: - SERP-фаза сохраняет только novostroyki-лоты — вторичкой владеет - cian_full_load. DETAIL/HOUSES-фазы не затронуты) - - cian_full_load → run_cian_full_load (exhaustive региональный сбор Cian ЕКБ ВТОРИЧКИ - без anchor'ов — room×price адаптивное партиционирование, - secondary_only, incremental on_bucket save; авторитетный источник - вторички. Окно 20:00-22:00 UTC чтобы не конфликтовать на shared - kf-прокси с cian_city_sweep (2-5) и cian_history_backfill (2-5)) - - domclick_city_sweep → run_domclick_city_sweep (scrape_pipeline.py; citywide SERP via - camoufox BrowserFetcher; shipped DORMANT enabled=false, enable - manually after prod-smoke) - - geoportal_coords_backfill → run_geoportal_coords_backfill - (tasks/backfill_listings_coords_geoportal.py, #1967; nightly - exact street+house match против ekb_geoportal_buildings (местный - реестр зданий ЕКБ) для listings.lat IS NULL. Pure internal DB op — - никакого внешнего HTTP/rate-limit (в отличие от Nominatim ниже). - Ранее был manual-only script (#1841); recurring нужен потому что - иначе новые NULL-coord листинги ждут geocode_missing_listings и - получают более грубый (coarse) city-centroid geocode вместо - точного house-level матча. Window 05:00-06:00 UTC — ДО - geocode_missing_listings (06:00-09:00 UTC), чтобы точный матч - имел приоритет над Nominatim.) - - geocode_missing_listings → run_geocode_missing_listings - (tasks/geocode_missing.py; nightly backfill listings.lat/lon для - всех источников с NULL coords; address-dedup batch loop с - wall-clock budget; window 06:00-09:00 UTC — после city sweeps - ~03-04 UTC, deals OS-cron 05:30/05:45 UTC и geoportal_coords_backfill - 05:00-06:00 UTC) - - yandex_detail_backfill → run_yandex_detail_backfill - (tasks/yandex_detail_backfill.py, #1553; nightly backfill of - yandex detail_enriched_at for ~3952 listings never enriched by - city_sweep; fetch via curl_cffi chrome120 + scraper_proxy_url, - parse via YandexDetailScraper.parse; - window 12:00-15:00 UTC — после avito_detail_backfill 09-12 UTC) - - cadastral_geo_match → run_cadastral_geo_match - (tasks/cadastral_geo_match.py; combined nightly job — (1) REFRESH - cad_buildings_local from gendesign_cad_buildings FDW (one bulk - scan, ~36.7k EKB rows, Point geom + GIST), (2) MATCH set-based - LATERAL KNN UPDATE filling listings.building_cadastral_number from - the nearest cadastral building within threshold_m (default 50m). - GEO-NEAREST approximation (nearest building, not exact cadastral). - Pure internal DB op — one FDW read + local UPDATE, no HTTP/anti-bot; - window 09:00-10:00 UTC — после geocode_missing_listings 06-09 UTC - чтобы listings имели свежие lat/lon/geom) - - house_imv_backfill → backfill_house_imv - (services/house_imv_backfill.py, #854; nightly bulk Avito IMV - на уровне домов — итерирует houses imv_status='pending' с lat/lon - + linked listings, считает медианные lot-params, зовёт - evaluate_via_imv, наполняет house_imv_evaluations / - house_placement_history / house_suggestions, переводит - houses.imv_status pending→ok/not_found/error. Resumable + - idempotent (UPSERT по house_id, фильтр imv_status). Avito IMV - использует собственную curl_cffi-сессию; window 15:00-17:00 UTC — - после avito_full_load 13-15 UTC и утреннего Avito-блока, дома - уже geocoded/enriched дневными sweep'ами) - - house_dedup_merge → run_house_dedup_merge - (services/house_dedup_merge.py, #1772; recurring, idempotent, - collision-safe house-deduplication merge. Clusters houses by - EXACT address (cadastral_number is 100% NULL), picks one canonical - keeper per cluster (geom NOT NULL → most listings → most-populated - → min id), re-points all FK children with UNIQUE-collision handling - — reuses migration 108's proven pattern — deletes loser rows, - backfills house_sources/house_address_aliases onto the keeper so the - matching pipeline finds the keeper next scrape. All in ONE - transaction; dry_run param computes counts then ROLLBACKs. - DESTRUCTIVE (DELETEs duplicate house rows) but idempotent: clean - table → empty mapping → 0-row no-op. Seed 135 ships the schedule - DISABLED (enabled=false) — deploy is neutral; enable deliberately - after a validated dry-run + manual run; weekly window 04:00-05:00 - UTC — quiet hour, no scraper/proxy contention) - - osm_poi_ekb_refresh → run_osm_poi_ekb_refresh - (tasks/osm_poi_ekb_refresh.py, #2045 BE-3; single bulk FDW scan — - TRUNCATE osm_poi_ekb_local; INSERT ... FROM gendesign_osm_poi_ekb - (Site Finder OSM POI, EKB), building Point(4326) geom from lat/lon. - Feeds app/services/location_coef.py (GET /trade-in/location-coef, - LocationDrawer). OSM POI changes rarely → daily cadence is ample. - Pure internal DB op — one FDW read + local TRUNCATE+INSERT, no - HTTP/anti-bot; window 10:00-11:00 UTC — после cadastral_geo_match - 09:00-10:00 UTC) +Что осталось в этом модуле — НЕ scheduler-loop, а функции с живыми потребителями вне +удалённой machinery: + - `compute_next_run_at` — читается admin.py (операторский предпросмотр "next run"). + - `has_running_run` — читается admin.py (UI-индикатор "уже бежит"). + - `import_rosreestr_dkp` — job-тело, вызываемое kit-handler'ом + product_handlers._job_rosreestr_dkp (lazy import). + - `_execute_cian_backfill` — job-тело, вызываемое kit-handler'ом + product_handlers._job_cian_history_backfill (lazy import). + +Zombie-reap, advisory-lock claim и tick-loop теперь целиком в +`scraper_kit/orchestration/scheduler.py` (см. test_scraper_kit_scheduler_parity.py). """ from __future__ import annotations -import asyncio import logging import random -from collections.abc import Coroutine from datetime import UTC, datetime, time, timedelta from typing import Any -import sentry_sdk from sqlalchemy import text from sqlalchemy.orm import Session -from app.core.db import SessionLocal from app.core.shutdown import shutdown_requested from app.services import scrape_runs as runs_mod -from app.services.scrape_pipeline import ( - run_avito_city_sweep, - run_avito_full_load, - run_avito_newbuilding_sweep, - run_cian_city_sweep, - run_cian_full_load, - run_domclick_city_sweep, - run_yandex_city_sweep, -) logger = logging.getLogger(__name__) -# Loop interval — check каждую минуту -SCHEDULER_TICK_SEC = 60 -ZOMBIE_THRESHOLD_HOURS = 6 - -# #1182 Phase 2: graceful SIGTERM-drain. Scrape-работа крутится в detached -# asyncio.create_task (см. _spawn_tracked) — coordinator (scheduler_loop) их -# НЕ join'ит. Без явного дренажа asyncio.run() teardown хард-кансельнул бы их -# mid-await (тот самый #1182 failure mode). Регистр strong-ref'ов даёт coordinator'у -# дождаться детей перед выходом из tick-loop'а. -_inflight_tasks: set[asyncio.Task[None]] = set() - -# Бюджет дренажа детей < scheduler_main._DRAIN_TIMEOUT_S (100s) < docker grace (120s): -# coordinator успевает дождаться кооперативных детей (avito_detail_backfill, -# rosreestr-executor дочекивают карточку/батч + mark_done и резолвятся) ВНУТРИ -# внешнего wait_for, а некооперативные упираются в этот timeout и падают на внешний -# hard-cancel fallback. -_CHILD_DRAIN_TIMEOUT_S = 80.0 - - -def _spawn_tracked(coro: Coroutine[Any, Any, None]) -> asyncio.Task[None]: - """create_task + регистрация задачи в _inflight_tasks для graceful-drain'а (#1182 P2). - - Заменяет прежний `task = create_task(_run()); task.add_done_callback(...)`: - strong-ref в set'е держит задачу до завершения (RUF006) и даёт scheduler_loop'у - дождаться её на SIGTERM-drain'е; done-callback ретривит exception (чтобы не было - 'Task exception was never retrieved') и убирает задачу из set'а. - """ - task = asyncio.create_task(coro) - _inflight_tasks.add(task) - - def _on_done(t: asyncio.Task[None]) -> None: - _inflight_tasks.discard(t) - if not t.cancelled(): - t.exception() - - task.add_done_callback(_on_done) - return task - - -async def _drain_inflight() -> None: - """Дождаться завершения detached run-задач перед teardown'ом процесса (#1182 P2). - - Зовётся из scheduler_loop при выходе из tick-loop'а по SIGTERM-drain'у. Кооперативные - дети дочекивают текущий unit (карточку/батч), делают mark_done и резолвятся; - некооперативные упираются в _CHILD_DRAIN_TIMEOUT_S и остаются на внешний hard-cancel - fallback (scheduler_main 100s + docker 120s). Не busy-spin — один asyncio.wait. - """ - pending = [t for t in _inflight_tasks if not t.done()] - if not pending: - return - logger.info("scheduler: draining %d in-flight run task(s) on shutdown", len(pending)) - _done, still = await asyncio.wait(pending, timeout=_CHILD_DRAIN_TIMEOUT_S) - if still: - logger.warning( - "scheduler: %d task(s) did not drain in %.0fs — leaving for hard-cancel", - len(still), - _CHILD_DRAIN_TIMEOUT_S, - ) - else: - logger.info("scheduler: all in-flight run task(s) drained cleanly") - def compute_next_run_at( window_start_hour: int, @@ -248,8 +48,8 @@ def compute_next_run_at( """Pick random datetime в window [start, end) UTC, через interval_days суток после now. interval_days задаёт каденс источника: 1 (default) = daily (back-compat), 7 = weekly. - Берётся из schedule.default_params["interval_days"] вызывающим кодом (_claim_run / - _defer_next_run_at); отсутствие ключа → 1 → прежнее ежедневное поведение. + Берётся из schedule.default_params["interval_days"] вызывающим кодом; отсутствие ключа + → 1 → прежнее ежедневное поведение. Если window_end_hour <= window_start_hour → cross-midnight window (например 22→3 → окно 22:00-23:59 ИЛИ 00:00-02:59). @@ -301,484 +101,6 @@ def has_running_run(db: Session, source: str) -> bool: return row is not None -def reap_zombies(db: Session) -> int: - """Mark scrape_runs as 'zombie' если heartbeat не обновлялся > ZOMBIE_THRESHOLD_HOURS hours.""" - zombie_interval = f"{ZOMBIE_THRESHOLD_HOURS} hours" - result = db.execute( - text( - """ - UPDATE scrape_runs - SET status = 'zombie', finished_at = NOW() - WHERE status = 'running' - AND (heartbeat_at IS NULL - OR heartbeat_at < NOW() - CAST(:interval AS interval)) - RETURNING id - """ - ), - {"interval": zombie_interval}, - ) - rows = result.fetchall() - db.commit() - if rows: - logger.warning("scheduler: reaped %d zombie runs: %s", len(rows), [r.id for r in rows]) - return len(rows) - - -def _claim_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Claim run: INSERT scrape_runs + UPDATE scrape_schedules next_run_at. - - Returns run_id или None если уже есть running run для этого source. - Общий helper для всех source-handler'ов. - """ - source = schedule_row["source"] - if has_running_run(db, source): - logger.info("scheduler: skip — already running for source=%s", source) - return None - - # #750: атомарный claim через transaction-scoped advisory lock. Сериализует - # конкурентные _claim_run одного source (redeploy-overlap старого+нового контейнера - # / 2 реплики), иначе оба пройдут has_running_run-pre-check и запустят ДВОЙНОЙ sweep - # → 2× rate к Avito/Cian → бан. pg_try_advisory_xact_lock НЕ блокирует: lock занят - # другим тиком → false → return None. Лок авто-снимается на commit/rollback этой - # транзакции (has_running_run выше — быстрый pre-check, обычно без взятия лока). - got_lock = db.execute( - text("SELECT pg_try_advisory_xact_lock(hashtext(:source))"), - {"source": source}, - ).scalar() - if not got_lock: - logger.info("scheduler: skip — concurrent claim in progress for source=%s", source) - return None - - # Double-checked под локом: конкурентный тик мог закоммитить running-run МЕЖДУ - # pre-check и взятием лока (затем освободить lock на commit). Перепроверяем под - # локом — иначе словим двойной run на узком окне. - if has_running_run(db, source): - logger.info("scheduler: skip — running appeared under lock for source=%s", source) - db.rollback() # освобождаем advisory lock (claim не состоялся) - return None - - params = schedule_row.get("default_params") or {} - run_id = runs_mod.create_run(db, source=source, params=params) - - next_at = compute_next_run_at( - schedule_row["window_start_hour"], - schedule_row["window_end_hour"], - interval_days=int(params.get("interval_days", 1)), - ) - db.execute( - text( - """ - UPDATE scrape_schedules - SET last_run_id = :run_id, last_run_at = NOW(), - next_run_at = :next_at, updated_at = NOW() - WHERE source = :source - """ - ), - {"run_id": run_id, "next_at": next_at, "source": source}, - ) - db.commit() - logger.info( - "scheduler: claimed run_id=%d source=%s next_run_at=%s", - run_id, - source, - next_at.isoformat(), - ) - return run_id - - -def _defer_next_run_at(db: Session, schedule_row: dict[str, Any]) -> None: - """Сдвинуть next_run_at на следующее окно БЕЗ создания run (#1522). - - Используется когда trigger делает early-return (например cian: отсутствуют/протухли - cookies) ДО _claim_run. Без этого get_due_schedules переотбирает schedule на каждом - тике (SCHEDULER_TICK_SEC), и pre-check (verify_session → реальный HTTP к Cian) гоняется - раз в минуту круглосуточно. Сдвиг откладывает следующую попытку до завтрашнего окна. - """ - source = schedule_row["source"] - params = schedule_row.get("default_params") or {} - next_at = compute_next_run_at( - schedule_row["window_start_hour"], - schedule_row["window_end_hour"], - interval_days=int(params.get("interval_days", 1)), - ) - db.execute( - text( - """ - UPDATE scrape_schedules - SET next_run_at = :next_at, updated_at = NOW() - WHERE source = :source - """ - ), - {"next_at": next_at, "source": source}, - ) - db.commit() - logger.info( - "scheduler: deferred next_run_at source=%s next_run_at=%s (no run claimed)", - source, - next_at.isoformat(), - ) - - -async def trigger_avito_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать новый scrape_runs + launch run_avito_city_sweep в asyncio.create_task. - - Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - # Spawn asyncio task — pass NEW session (avoid sharing with this scheduler tick) - async def _run() -> None: - run_db = SessionLocal() - try: - await run_avito_city_sweep( - run_db, - run_id=run_id, - pages_per_anchor=int(params.get("pages_per_anchor", 3)), - detail_top_n=int(params.get("detail_top_n", 20)), - request_delay_sec=float(params.get("request_delay_sec", 7.0)), - enrich_houses=bool(params.get("enrich_houses", True)), - radius_m=int(params.get("radius_m", 1500)), - ) - except Exception: - logger.exception("scheduler: run_avito_city_sweep crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered city_sweep run_id=%d", run_id) - return run_id - - -async def trigger_avito_newbuilding_sweep_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать новый scrape_runs + launch run_avito_newbuilding_sweep в asyncio.create_task. - - Зеркало trigger_avito_city_sweep_run, но citywide novostroyka-обход (без anchor/ - houses/detail-параметров): dedicated novostroyka-SERP → save. - - Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - # Spawn asyncio task — pass NEW session (avoid sharing with this scheduler tick) - async def _run() -> None: - run_db = SessionLocal() - try: - await run_avito_newbuilding_sweep( - run_db, - run_id=run_id, - pages=int(params.get("pages", 20)), - request_delay_sec=float(params.get("request_delay_sec", 7.0)), - ) - except Exception: - logger.exception("scheduler: run_avito_newbuilding_sweep crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered newbuilding_sweep run_id=%d", run_id) - return run_id - - -async def trigger_yandex_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать новый scrape_runs + launch run_yandex_city_sweep в asyncio.create_task. - - Зеркало trigger_avito_city_sweep_run, но проще (#561): у Yandex sweep нет - houses/detail-параметров. Shipped DORMANT — schedule seed enabled=false (078), - включается оператором вручную после #623 (egress-IP rotation). - - Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - # Spawn asyncio task — pass NEW session (avoid sharing with this scheduler tick) - async def _run() -> None: - run_db = SessionLocal() - try: - await run_yandex_city_sweep( - run_db, - run_id=run_id, - pages_per_anchor=int(params.get("pages_per_anchor", 2)), - request_delay_sec=float(params.get("request_delay_sec", 9.0)), - radius_m=int(params.get("radius_m", 1500)), - enrich_address=bool(params.get("enrich_address", True)), - segments=params.get("segments"), - ) - except Exception: - logger.exception("scheduler: run_yandex_city_sweep crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered yandex_city_sweep run_id=%d", run_id) - return run_id - - -async def trigger_cian_backfill_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_cian_backfill в asyncio.create_task. - - Pre-checks Cian session validity before claiming the run. - Marks nothing as failed if cookies are absent/expired — just logs + GlitchTip warning. - Returns run_id или None (skip / expired cookies). - """ - from app.services.cian_session import load_session, verify_session - - # Pre-check: load and verify cookies before claiming the run slot - cookies = load_session(db) - if cookies is None: - logger.warning("scheduler: cian_history_backfill skipped — no valid session cookies in DB") - # #1522: всё равно сдвигаем next_run_at на следующее окно, иначе get_due_schedules - # переотбирает этот schedule на КАЖДОМ тике (60с) и verify_session долбит Cian - # круглосуточно, пока оператор вручную не зальёт свежие cookies — burn proxy-квоты - # + риск TLS/bot-бана. - _defer_next_run_at(db, schedule_row) - return None - - state = await verify_session(cookies) - if state is None: - logger.warning( - "scheduler: cian_history_backfill — cookies expired or invalid, skipping run" - ) - try: - sentry_sdk.capture_message( - "cian_history_backfill skipped: Cian session cookies expired — " - "please re-upload via admin UI", - level="warning", - ) - except Exception: - pass # sentry_sdk not initialised in dev - # #1522: cookies протухли — сдвигаем next_run_at (см. комментарий выше), чтобы - # verify_session не бил по Cian каждые 60с до ручной перезаливки cookies. - _defer_next_run_at(db, schedule_row) - return None - - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - await _execute_cian_backfill(run_db, run_id=run_id, params=params) - except Exception: - logger.exception("scheduler: _execute_cian_backfill crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered cian_history_backfill run_id=%d", run_id) - return run_id - - -async def trigger_cian_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_cian_city_sweep в asyncio.create_task (#973). - - Несёт Phase 4 newbuilding-enrichment (enrich_houses). Shipped DORMANT — seed - enabled=false (107), включается оператором вручную после восстановления прокси. - Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - await run_cian_city_sweep( - run_db, - run_id=run_id, - pages_per_anchor=int(params.get("pages_per_anchor", 3)), - request_delay_sec=float(params.get("request_delay_sec", 5.0)), - radius_m=int(params.get("radius_m", 1500)), - detail_top_n=int(params.get("detail_top_n", 10)), - enrich_houses=bool(params.get("enrich_houses", True)), - newbuilding_only=bool(params.get("newbuilding_only", True)), - ) - except Exception: - logger.exception("scheduler: run_cian_city_sweep crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered cian_city_sweep run_id=%d", run_id) - return run_id - - -async def trigger_cian_full_load_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_cian_full_load в asyncio.create_task. - - Exhaustive региональный сбор Cian ЕКБ ВТОРИЧКИ (room×price партиционирование, - secondary_only, incremental on_bucket save) — авторитетный источник вторички. - Образец trigger_cian_city_sweep_run. Returns run_id (или None если skip — running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - await run_cian_full_load( - run_db, - run_id=run_id, - price_cap_per_bucket=int(params.get("price_cap_per_bucket", 1400)), - concurrency=int(params.get("concurrency", 5)), - request_delay_sec=float(params.get("request_delay_sec", 4.0)), - enrich_detail=bool(params.get("enrich_detail", False)), - detail_top_n=int(params.get("detail_top_n", 0)), - resume_run_id=None, - ) - except Exception: - logger.exception("scheduler: run_cian_full_load crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered cian_full_load run_id=%d", run_id) - return run_id - - -async def trigger_avito_full_load_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_avito_full_load в asyncio.create_task. - - Региональный сбор Avito ЕКБ ВТОРИЧКИ (room×price партиционирование через - pmin/pmax, secondary_only, incremental on_bucket save) — обходит SERP-cap - ~5000 на запрос. Зеркало trigger_cian_full_load_run. apw-прокси выбирает сам - scraper (AvitoScraper). - - Режим по умолчанию — инкрементальный: incremental_days читается из - default_params (shallow обход с date early-stop → избегает page-36 429-банов). - Для полного exhaustive обхода используется отдельный source - avito_full_load_exhaustive (incremental_days форсится в None). - - Returns run_id (или None если skip — running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - params = schedule_row.get("default_params") or {} - _incremental_days = params.get("incremental_days") - incremental_days = int(_incremental_days) if _incremental_days is not None else None - - async def _run() -> None: - run_db = SessionLocal() - try: - await run_avito_full_load( - run_db, - run_id=run_id, - price_cap_per_bucket=int(params.get("price_cap_per_bucket", 1400)), - concurrency=int(params.get("concurrency", 5)), - request_delay_sec=float(params.get("request_delay_sec", 7.0)), - secondary_only=bool(params.get("secondary_only", True)), - resume_run_id=None, - incremental_days=incremental_days, - ) - except Exception: - logger.exception("scheduler: run_avito_full_load crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered avito_full_load run_id=%d", run_id) - return run_id - - -async def trigger_avito_full_load_exhaustive_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать scrape_runs + launch run_avito_full_load (EXHAUSTIVE) в asyncio.create_task. - - Зеркало trigger_avito_full_load_run, но incremental_days форсится в None → - полный exhaustive обход (root-бисекция всего ценового диапазона на комнатность). - Запускается реже (weekly через interval_days=7 в default_params → compute_next_run_at - откладывает next_run_at на 7 суток) чтобы освежить last_seen (10-дневный delisting TTL) - и поймать тихие правки цены, которые shallow-инкремент пропускает. - Все прочие params (concurrency, price_cap_per_bucket, ...) читаются из - default_params как у avito_full_load. Returns run_id (или None если skip). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - await run_avito_full_load( - run_db, - run_id=run_id, - price_cap_per_bucket=int(params.get("price_cap_per_bucket", 1400)), - concurrency=int(params.get("concurrency", 5)), - request_delay_sec=float(params.get("request_delay_sec", 7.0)), - secondary_only=bool(params.get("secondary_only", True)), - resume_run_id=None, - incremental_days=None, # force exhaustive full walk - ) - except Exception: - logger.exception( - "scheduler: run_avito_full_load (exhaustive) crashed run_id=%d", run_id - ) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered avito_full_load_exhaustive run_id=%d", run_id) - return run_id - - -async def trigger_domclick_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_domclick_city_sweep в asyncio.create_task. - - Зеркало trigger_yandex_city_sweep_run, но citywide (city_id, без anchor/geo): - DomClick SERP не поддерживает geo-radius. Shipped DORMANT — seed enabled=false - (122), включается оператором вручную после prod-smoke. - - Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - await run_domclick_city_sweep( - run_db, - run_id=run_id, - city_id=int(params.get("city_id", 4)), - rooms=params.get("rooms"), - pages=int(params.get("pages_per_anchor", 5)), - request_delay_sec=float(params.get("request_delay_sec", 6.0)), - ) - except Exception: - logger.exception("scheduler: run_domclick_city_sweep crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered domclick_city_sweep run_id=%d", run_id) - return run_id - - async def _execute_cian_backfill( db: Session, *, @@ -845,554 +167,6 @@ async def _execute_cian_backfill( raise -async def trigger_yandex_address_backfill_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать scrape_runs + launch run_yandex_address_backfill в asyncio.create_task. - - EKB-only pilot (#855): backfills listings.address for yandex listings that have - street-only addresses (no house number). Fetches detail pages via curl_cffi chrome120, - extracts full address from <title>, updates listings WHERE address differs. - - Mirrors trigger_cian_backfill_run: claim run → create_task → mark_done/failed. - executor logic lives in tasks/yandex_address_backfill.py which delegates to - services/yandex_address_backfill.py (backfill_yandex_addresses). - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.yandex_address_backfill import run_yandex_address_backfill - - await run_yandex_address_backfill(run_db, run_id=run_id, params=params) - except Exception: - logger.exception("scheduler: run_yandex_address_backfill crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered yandex_address_backfill run_id=%d", run_id) - return run_id - - -async def trigger_geoportal_coords_backfill_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать scrape_runs + launch run_geoportal_coords_backfill в executor (sync DB-only task). - - Exact street+house match против ekb_geoportal_buildings (#1967 — was manual-only #1841). - Pure local DB op — никакого внешнего HTTP/rate-limit, поэтому запускается ПЕРЕД - geocode_missing_listings (Nominatim): точный house-level матч должен получить шанс - первым, иначе более грубый (coarse) geocode Nominatim'а забирает адрес раньше и - listing выпадает из WHERE lat IS NULL до следующего цикла. - - Sync task (SELECT + per-row local lookup + UPDATE, no async HTTP) — run in run_in_executor - by the same pattern as trigger_cadastral_geo_match_run. run_geoportal_coords_backfill owns - the scrape_runs lifecycle (mark_done/mark_failed). - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.backfill_listings_coords_geoportal import ( - run_geoportal_coords_backfill, - ) - - loop = asyncio.get_event_loop() - await loop.run_in_executor( - None, - lambda: run_geoportal_coords_backfill(run_db, run_id=run_id, params=params), - ) - except Exception: - logger.exception("scheduler: run_geoportal_coords_backfill crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered geoportal_coords_backfill run_id=%d", run_id) - return run_id - - -async def trigger_geocode_missing_listings_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать scrape_runs + launch run_geocode_missing_listings в asyncio.create_task. - - Nightly backfill listings.lat/lon для всех источников с NULL coords (#1: geom-coverage gap). - Адрес-dedup batch loop с wall-clock budget (default 30 мин). - - Root cause: scrapers сохраняют listings с lat=lon=NULL — «геокод подтянет cron», - но cron геокодирует только deals, не listings. Эта функция замыкает петлю: нигде - снаружи нет OS cron для listings — только этот планировщик. - - Mirrors trigger_yandex_address_backfill_run: claim run → create_task → mark_done/failed. - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.geocode_missing import run_geocode_missing_listings - - await run_geocode_missing_listings(run_db, run_id=run_id, params=params) - except Exception: - logger.exception("scheduler: run_geocode_missing_listings crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered geocode_missing_listings run_id=%d", run_id) - return run_id - - -async def trigger_rosreestr_dkp_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch import_rosreestr_dkp в executor (sync I/O-bound task).""" - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - # import_rosreestr_dkp is synchronous (DB-only, no async HTTP) - loop = asyncio.get_event_loop() - await loop.run_in_executor(None, import_rosreestr_dkp, run_db, run_id, params) - except Exception: - logger.exception("scheduler: import_rosreestr_dkp crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered rosreestr_dkp_import run_id=%d", run_id) - return run_id - - -async def trigger_listing_source_snapshot_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать scrape_runs + launch snapshot_listing_sources в executor (sync DB-only task). - - Per-source price-history daily snapshot (#570). Mirror trigger_rosreestr_dkp_run: - задача синхронная (чистый internal DB-snapshot, БЕЗ внешних HTTP-вызовов / анти-бота), - поэтому гоняем её в run_in_executor. SAFE to enable — schedule seed enabled=true (079), - в отличие от avito/yandex sweep (dormant). - - Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - async def _run() -> None: - run_db = SessionLocal() - try: - # snapshot_listing_sources is synchronous (DB-only, no async HTTP) - from app.tasks.listing_source_snapshot import snapshot_listing_sources - - loop = asyncio.get_event_loop() - await loop.run_in_executor(None, snapshot_listing_sources, run_db, run_id) - except Exception: - logger.exception("scheduler: snapshot_listing_sources crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered listing_source_snapshot run_id=%d", run_id) - return run_id - - -async def trigger_refresh_search_matview_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать scrape_runs + запустить REFRESH MATERIALIZED VIEW CONCURRENTLY listings_search_mv. - - listings_search_mv активно читается search_query.py (SELECT FROM listings_search_mv). - Без периодического refresh матвью устаревает по мере появления новых listings. - Задача синхронная (DB-only, БЕЗ внешних HTTP-вызовов) — гоняем в run_in_executor - по образцу trigger_asking_to_sold_ratio_run (#648). - - refresh_search_matview() использует собственное psycopg-соединение с autocommit=True - (REFRESH CONCURRENTLY не может работать внутри транзакции). scrape_runs lifecycle - (mark_done/mark_failed) управляется через отдельную SessionLocal. - - SAFE to enable — schedule seed enabled=true (088), pure internal DB op. - Окно 03:00-04:00 UTC — ночное, до rosreestr_dkp_import (04:00-06:00 UTC). - - Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.refresh_search_matview import refresh_search_matview - - loop = asyncio.get_event_loop() - # refresh_search_matview() manages its own connection with autocommit=True - # (required for REFRESH MATERIALIZED VIEW CONCURRENTLY outside a transaction). - await loop.run_in_executor(None, refresh_search_matview) - runs_mod.mark_done(run_db, run_id, {}) - except Exception: - logger.exception("scheduler: refresh_search_matview crashed run_id=%d", run_id) - runs_mod.mark_failed(run_db, run_id, "refresh_search_matview failed", {}) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered refresh_search_matview run_id=%d", run_id) - return run_id - - -async def trigger_deactivate_stale_avito_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать scrape_runs + launch deactivate_stale_avito_listings в executor (sync DB-only task). - - Помечает avito-объявления, не виденные >TTL дней, как is_active=false (#759). - Задача синхронная (чистый internal UPDATE, БЕЗ внешних HTTP-вызовов) — гоняем в - run_in_executor по образцу trigger_listing_source_snapshot_run. SAFE to enable — - schedule seed enabled=true (090), pure internal DB op. - Окно 06:00-07:00 UTC — после avito_city_sweep (02:00-05:00 UTC). - - Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.deactivate_stale_avito import deactivate_stale_avito_listings - - loop = asyncio.get_event_loop() - await loop.run_in_executor(None, deactivate_stale_avito_listings, run_db, run_id) - except Exception: - logger.exception("scheduler: deactivate_stale_avito crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered deactivate_stale_avito run_id=%d", run_id) - return run_id - - -async def trigger_deactivate_stale_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Универсальный триггер для всех deactivate_stale_* источников. - - Читает listing_source, ttl_days, segments, staleness_column из - schedule_row["default_params"] (jsonb). - Для источника 'deactivate_stale_avito' с пустым default_params использует defaults: - listing_source='avito', ttl_days=settings.avito_stale_ttl_days, segments=None, - staleness_column='last_seen_at'. - Для yandex/cian -- default_params обязан содержать listing_source + ttl_days + segments. - Для domklik (#2204) -- staleness_column='scraped_at' (нетрекаемый bulk-touch - двигает last_seen_at, честная свежесть = scraped_at). - - Запускает deactivate_stale_listings в executor (sync DB-only task). - Возвращает run_id или None (skip -- already running). - """ - from app.core.config import settings as _settings - - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - listing_source: str = params.get("listing_source", "avito") - ttl_days: int = params.get("ttl_days", _settings.avito_stale_ttl_days) - segments: list[str] | None = params.get("segments") # list or None - staleness_column: str = params.get("staleness_column", "last_seen_at") - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.deactivate_stale_avito import deactivate_stale_listings - - loop = asyncio.get_event_loop() - await loop.run_in_executor( - None, - lambda: deactivate_stale_listings( - run_db, - run_id, - listing_source=listing_source, - ttl_days=ttl_days, - segments=segments, - staleness_column=staleness_column, - ), - ) - except Exception: - logger.exception( - "scheduler: deactivate_stale source=%s crashed run_id=%d", - listing_source, - run_id, - ) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered deactivate_stale source=%s run_id=%d", listing_source, run_id) - return run_id - - -async def trigger_sber_index_pull_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_sber_index_pull в asyncio.create_task. - - Monthly pull (#887): fetches city-level secondary-housing price index from - sberindex.ru/api/sowa across three dashboards (real_estate_deals, - residential_real_estate_prices, dinamika-tsen-obyavlenii) and upserts into - sber_price_index. Pure HTTP+DB, no scraper proxy, no anti-bot. - - Mirrors trigger_yandex_address_backfill_run: claim run → create_task → mark_done/failed - delegated to tasks/sber_index_pull.py. - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.sber_index_pull import run_sber_index_pull - - await run_sber_index_pull(run_db, run_id=run_id, params=params) - except Exception: - logger.exception("scheduler: run_sber_index_pull crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered sber_index_pull run_id=%d", run_id) - return run_id - - -async def trigger_rosreestr_quarter_poll_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать scrape_runs + launch run_rosreestr_quarter_poll в asyncio.create_task. - - Monthly HEAD-check (#888): polls rosreestr.gov.ru for the next quarter's open-data - ZIP to detect when a new quarter is published. Pure HTTP detect+alert — does NOT - download the multi-GB dataset or shell out to the loaders. When available=True, a - clear actionable INFO log is emitted for the operator. - - Mirrors trigger_sber_index_pull_run: claim run → create_task → mark_done/failed - delegated to tasks/rosreestr_quarter_poll.py. - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.rosreestr_quarter_poll import run_rosreestr_quarter_poll - - await run_rosreestr_quarter_poll(run_db, run_id=run_id, params=params) - except Exception: - logger.exception("scheduler: run_rosreestr_quarter_poll crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered rosreestr_quarter_poll run_id=%d", run_id) - return run_id - - -async def trigger_deals_freshness_monitor_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать scrape_runs + launch check_deals_freshness в executor (sync DB-only task). - - Мониторинг свежести ДАННЫХ deals (#2212): смотрит max(deal_date) и поднимает - ERROR-алерт (→ GlitchTip), когда следующий квартал просрочен публикацией. Джоба- - статус rosreestr_* остаётся зелёным при staleness данных — этот монитор ловит - именно устаревание данных, а не сбой прогона. - - Задача синхронная (один SELECT max(deal_date), без внешних HTTP) — гоняем в - run_in_executor по образцу trigger_listing_source_snapshot_run. - - Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.deals_freshness_monitor import check_deals_freshness - - loop = asyncio.get_event_loop() - await loop.run_in_executor(None, check_deals_freshness, run_db, run_id, params) - except Exception: - logger.exception("scheduler: check_deals_freshness crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered deals_freshness_monitor run_id=%d", run_id) - return run_id - - -async def trigger_newbuilding_enrich_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_newbuilding_enrich в asyncio.create_task (#973). - - Nightly slice of the cian_newbuilding enrichment backfill: resolves cian_zhk_url from - house_sources.ext_id when missing, fetches the ЖК page (curl_cffi chrome120) and persists - houses_price_dynamics / house_reliability_checks / house_reviews. This is the standalone - schedule the inline cian full-sweep path never had — 306 fetchable cian houses on prod, - only ~3 enriched so far, so the backlog drains over successive nights. - - Mirrors trigger_sber_index_pull_run: claim run → create_task → mark_done/failed delegated - to tasks/newbuilding_enrich_backfill.run_newbuilding_enrich (which owns the run lifecycle). - - Idempotency: run_newbuilding_enrich → backfill_newbuilding_enrichment skips houses that - already have BOTH price_dynamics AND reliability rows (force=False) and uses a per-house - SAVEPOINT, so a re-fire never re-enriches done houses or double-appends on a partial run. - `limit` (default_params) bounds how many houses one fire fetches — anti-bot politeness; - the nightly cadence drains the rest. zombie-reap is inherited from the shared loop. - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.newbuilding_enrich_backfill import run_newbuilding_enrich - - await run_newbuilding_enrich(run_db, run_id=run_id, params=params) - except Exception: - logger.exception("scheduler: run_newbuilding_enrich crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered newbuilding_enrich run_id=%d", run_id) - return run_id - - -async def trigger_yandex_newbuilding_sweep_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Создать scrape_runs + launch enrich_yandex_newbuilding_sweep в asyncio.create_task (#974). - - Nightly enrichment sweep: SELECT pending yandex_realty_nb houses → resolve slug - (BrowserFetcher SERP) → fetch_jk (BrowserFetcher) → UPSERT market.yandex_jk_enrichment. - - Shipped DORMANT (seed 106, enabled=false). Включается оператором вручную: - UPDATE scrape_schedules SET enabled = true WHERE source = 'yandex_newbuilding_sweep'; - - Mirrors trigger_newbuilding_enrich_run: claim run → create_task → mark_done/failed - делегируется tasks/yandex_newbuilding_sweep.enrich_yandex_newbuilding_sweep. - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.yandex_newbuilding_sweep import enrich_yandex_newbuilding_sweep - - result = await enrich_yandex_newbuilding_sweep( - run_db, - limit=int(params.get("limit", 5)), - request_delay_sec=float(params.get("request_delay_sec", 8.0)), - city=str(params.get("city", "ekaterinburg")), - ) - runs_mod.mark_done(run_db, run_id, result.to_dict()) - except Exception: - logger.exception("scheduler: enrich_yandex_newbuilding_sweep crashed run_id=%d", run_id) - try: - runs_mod.mark_failed(run_db, run_id, "crashed", {}) - except Exception: - pass - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered yandex_newbuilding_sweep run_id=%d", run_id) - return run_id - - -async def trigger_asking_to_sold_ratio_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch recompute_asking_to_sold_ratios в executor (sync DB-only task). - - Daily TRUE-MIRROR refresh асking→sold коэффициентов (#648 Stage 4): DELETE WHERE - district='' + re-derive INSERT (та же 080-derivation) в одной транзакции, чтобы ratio - не устаревал по мере ночного импорта ДКП. Mirror trigger_listing_source_snapshot_run: - задача синхронная (чистый internal DB re-derive, БЕЗ внешних HTTP-вызовов / анти-бота), - поэтому гоняем её в run_in_executor. SAFE to enable — schedule seed enabled=true (082), - в отличие от avito/yandex sweep (dormant). - - Returns run_id (или None если skip — есть running run). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - async def _run() -> None: - run_db = SessionLocal() - try: - # recompute_asking_to_sold_ratios is synchronous (DB-only, no async HTTP) - from app.tasks.asking_to_sold_ratio import recompute_asking_to_sold_ratios - - loop = asyncio.get_event_loop() - await loop.run_in_executor(None, recompute_asking_to_sold_ratios, run_db, run_id) - except Exception: - logger.exception("scheduler: recompute_asking_to_sold_ratios crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered asking_to_sold_ratio_refresh run_id=%d", run_id) - return run_id - - def import_rosreestr_dkp( db: Session, run_id: int, @@ -1642,457 +416,3 @@ def import_rosreestr_dkp( logger.exception("rosreestr_dkp_import run_id=%d failed at last_id=%d", run_id, last_id) runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters) raise - - -async def trigger_avito_detail_backfill_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Create scrape_run + launch run_avito_detail_backfill in asyncio.create_task. - - Nightly backfill detail_enriched_at for legacy avito listings (#1551). - ~15243/15917 avito listings have detail_enriched_at IS NULL -- never enriched - by city_sweep ENRICH_DETAIL (only top-N proximate per anchor are touched). - - Mirrors trigger_geocode_missing_listings_run: claim run -> create_task -> - mark_done/failed delegated to tasks/avito_detail_backfill.run_avito_detail_backfill. - - Returns run_id or None (skip -- already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.avito_detail_backfill import run_avito_detail_backfill - - await run_avito_detail_backfill(run_db, run_id=run_id, params=params) - except Exception: - logger.exception("scheduler: run_avito_detail_backfill crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered avito_detail_backfill run_id=%d", run_id) - return run_id - - -async def trigger_yandex_detail_backfill_run( - db: Session, schedule_row: dict[str, Any] -) -> int | None: - """Create scrape_run + launch run_yandex_detail_backfill in asyncio.create_task. - - Nightly backfill detail_enriched_at for legacy yandex listings (#1553). - ~3952 yandex listings have detail_enriched_at IS NULL -- SERP-only data, - never enriched by YandexDetailScraper. - - Mirrors trigger_avito_detail_backfill_run: claim run -> create_task -> - mark_done/failed delegated to tasks/yandex_detail_backfill.run_yandex_detail_backfill. - - Returns run_id or None (skip -- already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.yandex_detail_backfill import run_yandex_detail_backfill - - await run_yandex_detail_backfill(run_db, run_id=run_id, params=params) - except Exception: - logger.exception("scheduler: run_yandex_detail_backfill crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered yandex_detail_backfill run_id=%d", run_id) - return run_id - - -async def trigger_cadastral_geo_match_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_cadastral_geo_match в executor (sync DB-only task). - - Combined refresh+match (#cadastral-geo-match): (1) REFRESH cad_buildings_local from the - gendesign_cad_buildings FDW (one bulk scan, builds Point geom), (2) MATCH set-based - LATERAL KNN UPDATE filling listings.building_cadastral_number from the nearest cadastral - building within threshold_m (default 50m). GEO-NEAREST approximation. - - Sync task (one FDW read + local UPDATE, no async HTTP) — run in run_in_executor by the - same pattern as trigger_listing_source_snapshot_run / trigger_asking_to_sold_ratio_run. - run_cadastral_geo_match owns the scrape_runs lifecycle (mark_done/mark_failed). SAFE to - enable — seed 125 enabled=true, pure internal DB op. - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.cadastral_geo_match import run_cadastral_geo_match - - loop = asyncio.get_event_loop() - await loop.run_in_executor( - None, - lambda: run_cadastral_geo_match(run_db, run_id=run_id, params=params), - ) - except Exception: - logger.exception("scheduler: run_cadastral_geo_match crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered cadastral_geo_match run_id=%d", run_id) - return run_id - - -async def trigger_osm_poi_ekb_refresh_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_osm_poi_ekb_refresh в executor (#2045 BE-3). - - Single bulk FDW scan (TRUNCATE osm_poi_ekb_local; INSERT ... FROM gendesign_osm_poi_ekb) — - feeds app/services/location_coef.py. No params consumed. - - Sync task (one FDW read + local TRUNCATE+INSERT, no async HTTP) — run in run_in_executor - by the same pattern as trigger_cadastral_geo_match_run. run_osm_poi_ekb_refresh owns the - scrape_runs lifecycle (mark_done/mark_failed). SAFE to enable — seed 170 enabled=true, - pure internal DB op. - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.tasks.osm_poi_ekb_refresh import run_osm_poi_ekb_refresh - - loop = asyncio.get_event_loop() - await loop.run_in_executor( - None, - lambda: run_osm_poi_ekb_refresh(run_db, run_id=run_id, params=params), - ) - except Exception: - logger.exception("scheduler: run_osm_poi_ekb_refresh crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered osm_poi_ekb_refresh run_id=%d", run_id) - return run_id - - -async def trigger_house_imv_backfill_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch backfill_house_imv в asyncio.create_task (#854). - - Bulk Avito IMV-оценка на уровне домов: итерирует houses с imv_status=:only_status - (default 'pending'), у которых есть lat/lon + хотя бы один listing с rooms+area, - считает медианные lot-params, зовёт evaluate_via_imv и наполняет - house_imv_evaluations + house_placement_history + house_suggestions (3×UPSERT), - переводя houses.imv_status pending→ok/not_found/error/transient_error. - - Зеркало trigger_avito_detail_backfill_run / trigger_yandex_address_backfill_run: - _claim_run → asyncio.create_task → fresh SessionLocal → add_done_callback (RUF006). - Отличие: у house_imv_backfill нет tasks/*-обёртки, владеющей lifecycle, поэтому - финализацию scrape_runs (mark_done/mark_failed) делаем здесь, как в - trigger_refresh_search_matview_run. heartbeat-колбэк дёргает update_heartbeat, - чтобы reap_zombies не пометил живой долгий IMV-run 'zombie' (#1363). - - Resumable + idempotent: backfill_house_imv фильтрует imv_status и UPSERT'ит по - house_id, поэтому повторный fire дренит следующий batch без дублей. - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - batch_size = int(params.get("batch_size", 50)) - request_delay_sec = float(params.get("request_delay_sec", 5.0)) - only_status = str(params.get("only_status", "pending")) - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.services.house_imv_backfill import backfill_house_imv - - # heartbeat дёргается каждые N домов внутри backfill — обновляем - # scrape_runs.heartbeat_at, иначе reap_zombies снимет живой run (#1363). - def _heartbeat() -> None: - runs_mod.update_heartbeat(run_db, run_id, {}) - - result = await backfill_house_imv( - run_db, - batch_size=batch_size, - request_delay_sec=request_delay_sec, - only_status=only_status, - heartbeat=_heartbeat, - ) - runs_mod.mark_done( - run_db, - run_id, - { - "checked": result.checked, - "saved": result.saved, - "skipped": result.skipped, - "errors": result.errors, - "duration_sec": int(result.duration_sec), - }, - ) - except Exception as exc: - logger.exception("scheduler: backfill_house_imv crashed run_id=%d", run_id) - try: - runs_mod.mark_failed(run_db, run_id, str(exc)[:1000], {}) - except Exception: - logger.exception("scheduler: mark_failed crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered house_imv_backfill run_id=%d", run_id) - return run_id - - -async def trigger_house_dedup_merge_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_house_dedup_merge в executor (sync DB-only task, #1772). - - Recurring, idempotent, collision-safe house-deduplication merge: clusters `houses` by - EXACT address, picks one canonical keeper per cluster (geom NOT NULL → most listings → - most-populated → min id), re-points all FK children with UNIQUE-collision handling - (reuses migration 108's proven pattern), deletes the loser rows, backfills - house_sources/house_address_aliases onto the keeper — all in ONE transaction. - - DESTRUCTIVE (DELETEs duplicate house rows) but idempotent: a clean table yields an empty - mapping → every statement is a 0-row no-op. Seed 135 ships the schedule DISABLED - (enabled=false) — the deploy is neutral; the orchestrator dry-runs + does one validated - manual run, then enables deliberately. default_params.dry_run=true previews without writes. - - Sync task (set-based UPDATE/DELETE, no async HTTP) — run in run_in_executor by the same - pattern as trigger_cadastral_geo_match_run. run_house_dedup_merge owns the scrape_runs - lifecycle (mark_done/mark_failed). - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.services.house_dedup_merge import run_house_dedup_merge - - loop = asyncio.get_event_loop() - await loop.run_in_executor( - None, - lambda: run_house_dedup_merge(run_db, run_id=run_id, params=params), - ) - except Exception: - logger.exception("scheduler: run_house_dedup_merge crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered house_dedup_merge run_id=%d", run_id) - return run_id - - -async def trigger_proxy_healthcheck_run(db: Session, schedule_row: dict[str, Any]) -> int | None: - """Создать scrape_runs + launch run_proxy_healthcheck в asyncio.create_task (#2162). - - Периодический health-check пула scrape_proxies: reap_stale_leases + ipify-проба через - каждый enabled-прокси → mark_health (сброс fails/exit_ip/latency при успехе; инкремент - + авто-disable при DISABLE_THRESHOLD на фейле). АДДИТИВНО — не трогает боевые скрейперы. - - Зеркало trigger_yandex_newbuilding_sweep_run: у run_proxy_healthcheck нет tasks/*-обёртки, - владеющей lifecycle, поэтому финализацию scrape_runs (mark_done/mark_failed) делаем здесь. - - Каденс: _claim_run ставит next_run_at через compute_next_run_at (суточная гранулярность), - но health-check хочется чаще. Поэтому СРАЗУ после claim переопределяем next_run_at на - now() + interval_minutes (default 30) — source-специфичный sub-hourly re-schedule, не - трогая shared compute_next_run_at. Если run ещё идёт на следующем тике — has_running_run - в _claim_run вернёт None (skip) и next_run_at не сбросится, т.е. авто-throttle по факту. - - Returns run_id или None (skip — already running). - """ - run_id = _claim_run(db, schedule_row) - if run_id is None: - return None - - params = schedule_row.get("default_params") or {} - interval_minutes = int(params.get("interval_minutes", 30)) - db.execute( - text( - """ - UPDATE scrape_schedules - SET next_run_at = now() + make_interval(mins => CAST(:mins AS integer)), - updated_at = now() - WHERE source = 'proxy_healthcheck' - """ - ), - {"mins": interval_minutes}, - ) - db.commit() - - async def _run() -> None: - run_db = SessionLocal() - try: - from app.services.proxy_pool import run_proxy_healthcheck - - counters = await run_proxy_healthcheck(run_db) - runs_mod.mark_done(run_db, run_id, counters) - except Exception as exc: - logger.exception("scheduler: run_proxy_healthcheck crashed run_id=%d", run_id) - try: - runs_mod.mark_failed(run_db, run_id, str(exc)[:1000], {}) - except Exception: - logger.exception("scheduler: mark_failed crashed run_id=%d", run_id) - finally: - run_db.close() - - _spawn_tracked(_run()) - logger.info("scheduler: triggered proxy_healthcheck run_id=%d", run_id) - return run_id - - -def get_due_schedules(db: Session) -> list[dict[str, Any]]: - """SELECT scrape_schedules WHERE enabled AND (next_run_at IS NULL OR next_run_at <= NOW()).""" - rows = ( - db.execute( - text( - """ - SELECT id, source, enabled, window_start_hour, window_end_hour, - default_params, last_run_id, last_run_at, next_run_at - FROM scrape_schedules - WHERE enabled = true - AND (next_run_at IS NULL OR next_run_at <= NOW()) - """ - ), - ) - .mappings() - .all() - ) - return [dict(r) for r in rows] - - -async def scheduler_loop() -> None: - """Бесконечный async loop — tick каждые SCHEDULER_TICK_SEC секунд.""" - logger.info("scheduler: started (tick=%ds)", SCHEDULER_TICK_SEC) - # Initial sleep 30s чтобы дать FastAPI startup завершиться - await asyncio.sleep(30) - while True: - # #1182 Phase 2: кооперативный SIGTERM-drain. Не reap'аем и не claim'аем - # новые run'ы во время shutdown — даём текущему dispatch'у докатиться и выходим. - if shutdown_requested(): - logger.info("scheduler: SIGTERM-drain — stop claiming new runs, exiting tick loop") - break - try: - db = SessionLocal() - try: - # Reap zombies first - reap_zombies(db) - # Process due schedules - due = get_due_schedules(db) - for sch in due: - source = sch["source"] - if source == "avito_city_sweep": - await trigger_avito_city_sweep_run(db, sch) - elif source == "avito_full_load": - await trigger_avito_full_load_run(db, sch) - elif source == "avito_full_load_exhaustive": - await trigger_avito_full_load_exhaustive_run(db, sch) - elif source == "avito_newbuilding_sweep": - await trigger_avito_newbuilding_sweep_run(db, sch) - elif source == "yandex_city_sweep": - await trigger_yandex_city_sweep_run(db, sch) - elif source == "cian_history_backfill": - await trigger_cian_backfill_run(db, sch) - elif source == "cian_city_sweep": - await trigger_cian_city_sweep_run(db, sch) - elif source == "cian_full_load": - await trigger_cian_full_load_run(db, sch) - elif source == "domclick_city_sweep": - await trigger_domclick_city_sweep_run(db, sch) - elif source == "rosreestr_dkp_import": - await trigger_rosreestr_dkp_run(db, sch) - elif source == "listing_source_snapshot": - await trigger_listing_source_snapshot_run(db, sch) - elif source == "asking_to_sold_ratio_refresh": - await trigger_asking_to_sold_ratio_run(db, sch) - elif source == "refresh_search_matview": - await trigger_refresh_search_matview_run(db, sch) - elif source == "yandex_address_backfill": - await trigger_yandex_address_backfill_run(db, sch) - elif source.startswith("deactivate_stale_"): - await trigger_deactivate_stale_run(db, sch) - elif source == "sber_index_pull": - await trigger_sber_index_pull_run(db, sch) - elif source == "rosreestr_quarter_poll": - await trigger_rosreestr_quarter_poll_run(db, sch) - elif source == "deals_freshness_monitor": - await trigger_deals_freshness_monitor_run(db, sch) - elif source == "newbuilding_enrich": - await trigger_newbuilding_enrich_run(db, sch) - elif source == "yandex_newbuilding_sweep": - await trigger_yandex_newbuilding_sweep_run(db, sch) - elif source == "geoportal_coords_backfill": - await trigger_geoportal_coords_backfill_run(db, sch) - elif source == "geocode_missing_listings": - await trigger_geocode_missing_listings_run(db, sch) - elif source == "avito_detail_backfill": - await trigger_avito_detail_backfill_run(db, sch) - elif source == "yandex_detail_backfill": - await trigger_yandex_detail_backfill_run(db, sch) - elif source == "cadastral_geo_match": - await trigger_cadastral_geo_match_run(db, sch) - elif source == "osm_poi_ekb_refresh": - await trigger_osm_poi_ekb_refresh_run(db, sch) - elif source == "house_imv_backfill": - await trigger_house_imv_backfill_run(db, sch) - elif source == "house_dedup_merge": - await trigger_house_dedup_merge_run(db, sch) - elif source == "proxy_healthcheck": - await trigger_proxy_healthcheck_run(db, sch) - else: - logger.warning("scheduler: unknown source=%s, skip", source) - - # #1182 Phase 2: после каждого dispatch'а проверяем drain — текущий - # run уже отпущен в свою asyncio-задачу (сам докоммитит/выйдет по - # своему checkpoint'у), а новые в этом тике не запускаем. - if shutdown_requested(): - logger.info( - "scheduler: SIGTERM-drain — stop dispatch after source=%s", source - ) - break - finally: - db.close() - except Exception: - logger.exception("scheduler: tick failed") - # Промежуточная проверка перед 60s-сном: выходим сразу, не ждём целый тик. - if shutdown_requested(): - logger.info("scheduler: SIGTERM-drain — exiting tick loop after dispatch") - break - await asyncio.sleep(SCHEDULER_TICK_SEC) - - # Tick-loop вышел только по SIGTERM-drain'у (иначе while True бесконечен): дожидаемся - # detached run-задач (_spawn_tracked) — пусть докоммитят текущий unit и сделают - # mark_done, а не дадим asyncio.run() teardown'у хард-кансельнуть их mid-await (#1182). - await _drain_inflight() - logger.info("scheduler: tick loop exited (in-flight drain complete)") diff --git a/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py b/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py index b21ad247..7b588372 100644 --- a/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py +++ b/tradein-mvp/backend/app/tasks/asking_to_sold_ratio.py @@ -13,9 +13,9 @@ TRUE-MIRROR REFRESH (флаг database-expert): 080-seed использовал заново гоняет ту же 080-derivation INSERT...SELECT. DELETE+INSERT в ОДНОЙ транзакции (атомарно — таблица никогда не пуста mid-refresh). После commit таблица == свежий re-seed. -Задача синхронная (DB-only, никаких внешних HTTP-вызовов) — запускается in-app -scheduler'ом через trigger_asking_to_sold_ratio_run() (scheduler.py), по образцу -snapshot_listing_sources / import_rosreestr_dkp (sync task в run_in_executor). +Задача синхронная (DB-only, никаких внешних HTTP-вызовов) — запускается kit-scheduler'ом +через product_handlers._job_asking_to_sold_ratio (run_in_executor), по образцу +snapshot_listing_sources / import_rosreestr_dkp. Окно расписания 06:00-07:00 UTC — ПОСЛЕ rosreestr_dkp_import (04:00-06:00 UTC), чтобы refresh потреблял свежие ДКП-сделки того же дня. diff --git a/tradein-mvp/backend/app/tasks/backfill_listings_coords_geoportal.py b/tradein-mvp/backend/app/tasks/backfill_listings_coords_geoportal.py index b67a234f..035f7be1 100644 --- a/tradein-mvp/backend/app/tasks/backfill_listings_coords_geoportal.py +++ b/tradein-mvp/backend/app/tasks/backfill_listings_coords_geoportal.py @@ -281,11 +281,11 @@ def run_geoportal_coords_backfill( ) -> BackfillCoordsResult: """Run-lifecycle wrapper for backfill_coords_from_geoportal (sync, DB-only task). - Launched by the in-app scheduler (source='geoportal_coords_backfill') via - trigger_geoportal_coords_backfill_run, или вручную. Pure local exact street+house match - против ekb_geoportal_buildings — никакого внешнего HTTP, поэтому весь backlog (или - ограниченный `limit`) отрабатывается за один прогон без wall-clock budget (в отличие - от geocode_missing_listings, который rate-limited Nominatim'ом). + Launched by the kit scheduler (source='geoportal_coords_backfill') via + product_handlers._job_geoportal_coords_backfill, или вручную. Pure local exact + street+house match против ekb_geoportal_buildings — никакого внешнего HTTP, поэтому + весь backlog (или ограниченный `limit`) отрабатывается за один прогон без wall-clock + budget (в отличие от geocode_missing_listings, который rate-limited Nominatim'ом). Params (default_params jsonb): limit: максимум листингов за прогон (default None = весь backlog). diff --git a/tradein-mvp/backend/app/tasks/cadastral_geo_match.py b/tradein-mvp/backend/app/tasks/cadastral_geo_match.py index 2721cdc2..3bb58c55 100644 --- a/tradein-mvp/backend/app/tasks/cadastral_geo_match.py +++ b/tradein-mvp/backend/app/tasks/cadastral_geo_match.py @@ -267,10 +267,10 @@ def refresh_and_match( def run_cadastral_geo_match(db: Session, *, run_id: int, params: dict) -> CadMatchResult: """Run-lifecycle wrapper for the combined refresh+match job (sync, DB-only). - Launched by the in-app scheduler (source='cadastral_geo_match') via - trigger_cadastral_geo_match_run, or manually. Refreshes cad_buildings_local from the - FDW (one bulk scan) then geo-matches listings.building_cadastral_number to the nearest - cadastral building within threshold_m. + Launched by the kit scheduler (source='cadastral_geo_match') via + product_handlers._job_cadastral_geo_match, or manually. Refreshes cad_buildings_local + from the FDW (one bulk scan) then geo-matches listings.building_cadastral_number to the + nearest cadastral building within threshold_m. Params (default_params jsonb): threshold_m: max nearest-building distance to accept (metres, default 50). diff --git a/tradein-mvp/backend/app/tasks/deactivate_stale_avito.py b/tradein-mvp/backend/app/tasks/deactivate_stale_avito.py index 6d8baf1b..1d1c2188 100644 --- a/tradein-mvp/backend/app/tasks/deactivate_stale_avito.py +++ b/tradein-mvp/backend/app/tasks/deactivate_stale_avito.py @@ -10,8 +10,8 @@ - avito: все сегменты (segments=None), TTL=10 дней -- поведение без изменений. - Строки НЕ удаляются -- история нужна для бэктеста (#667). -Задача синхронная (DB-only, никаких внешних HTTP-вызовов) -- запускается in-app -scheduler'ом через trigger_deactivate_stale_run() (scheduler.py), +Задача синхронная (DB-only, никаких внешних HTTP-вызовов) -- запускается kit-scheduler'ом +через product_handlers._job_deactivate_stale (wildcard-handler deactivate_stale_*), по образцу snapshot_listing_sources / recompute_asking_to_sold_ratios (sync task в run_in_executor). diff --git a/tradein-mvp/backend/app/tasks/deals_freshness_monitor.py b/tradein-mvp/backend/app/tasks/deals_freshness_monitor.py index 0a687913..b585551c 100644 --- a/tradein-mvp/backend/app/tasks/deals_freshness_monitor.py +++ b/tradein-mvp/backend/app/tasks/deals_freshness_monitor.py @@ -19,8 +19,8 @@ sentry logging integration), когда следующий квартал ПРО LAG_ALLOWANCE_DAYS (по умолчанию 45) на публикацию дампа Росреестром. Сейчас (2026-07-02) до порога 2026-08-15 ещё далеко → алерта НЕТ. Если к 2026-08-16 Q2 не импортирован → алерт. -Задача синхронная (DB-only, один SELECT max(deal_date)) — запускается in-app scheduler'ом -через trigger_deals_freshness_monitor_run() в run_in_executor, по образцу +Задача синхронная (DB-only, один SELECT max(deal_date)) — запускается kit-scheduler'ом +через product_handlers._job_deals_freshness_monitor в run_in_executor, по образцу listing_source_snapshot / rosreestr_dkp_import. Вердикт вычисляет ЧИСТАЯ функция evaluate_deals_freshness() (frozen-now тестируется без БД). diff --git a/tradein-mvp/backend/app/tasks/listing_source_snapshot.py b/tradein-mvp/backend/app/tasks/listing_source_snapshot.py index 2516fd02..06a19ca5 100644 --- a/tradein-mvp/backend/app/tasks/listing_source_snapshot.py +++ b/tradein-mvp/backend/app/tasks/listing_source_snapshot.py @@ -5,8 +5,8 @@ listing_source_events. Так история per-source цены копится СРАЗУ, независимо от (сейчас DORMANT) скраперов — см. шапку data/sql/079_listing_source_history.sql. -Задача синхронная (DB-only, никаких внешних HTTP-вызовов) — запускается in-app -scheduler'ом через trigger_listing_source_snapshot_run() (scheduler.py), +Задача синхронная (DB-only, никаких внешних HTTP-вызовов) — запускается kit-scheduler'ом +через product_handlers._job_listing_source_snapshot, по образцу import_rosreestr_dkp (sync task в run_in_executor). Вся работа — два set-based SQL statement'а (snapshot upsert + event-diff CTE), diff --git a/tradein-mvp/backend/app/tasks/osm_poi_ekb_refresh.py b/tradein-mvp/backend/app/tasks/osm_poi_ekb_refresh.py index 246eaced..1b0d2ee6 100644 --- a/tradein-mvp/backend/app/tasks/osm_poi_ekb_refresh.py +++ b/tradein-mvp/backend/app/tasks/osm_poi_ekb_refresh.py @@ -75,9 +75,9 @@ class OsmPoiRefreshResult: def run_osm_poi_ekb_refresh(db: Session, *, run_id: int, params: dict) -> OsmPoiRefreshResult: """Run-lifecycle wrapper for the OSM POI local-mirror refresh (scheduler entrypoint). - Launched by the in-app scheduler (source='osm_poi_ekb_refresh') via - trigger_osm_poi_ekb_refresh_run, or manually. No params consumed — single unconditional - bulk refresh (`params` accepted for scheduler-signature compatibility / future use). + Launched by the kit scheduler (source='osm_poi_ekb_refresh') via + product_handlers._job_osm_poi_ekb_refresh, or manually. No params consumed — single + unconditional bulk refresh (`params` accepted for scheduler-signature compatibility). Finalises scrape_runs (mark_done / mark_failed) with counters. """ diff --git a/tradein-mvp/backend/tests/tasks/test_backfill_listings_coords_geoportal.py b/tradein-mvp/backend/tests/tasks/test_backfill_listings_coords_geoportal.py index 6397e2c5..c2db0780 100644 --- a/tradein-mvp/backend/tests/tasks/test_backfill_listings_coords_geoportal.py +++ b/tradein-mvp/backend/tests/tasks/test_backfill_listings_coords_geoportal.py @@ -10,14 +10,13 @@ - Идемпотентность: UPDATE с WHERE lat IS NULL не затирает уже заполненное. - SAVEPOINT откат батча при исключении в DB.execute. - run_geoportal_coords_backfill (#1967): run-lifecycle wrapper mark_done/mark_failed. -- Scheduler wiring (#1967): trigger fn + dispatch branch + migration 171 seed content. +- Migration 171 seed content. Все сетевые вызовы и DB замоканы. """ from __future__ import annotations -import inspect import os import sys from pathlib import Path @@ -31,7 +30,6 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost: _wp_mock = MagicMock() sys.modules.setdefault("weasyprint", _wp_mock) -from app.services import scheduler # noqa: E402 from app.services.geocoder import GeocodeSuggestion # noqa: E402 from app.tasks import backfill_listings_coords_geoportal as bcg # noqa: E402 from app.tasks.backfill_listings_coords_geoportal import ( # noqa: E402 @@ -437,38 +435,6 @@ def test_run_wrapper_marks_failed_on_error(monkeypatch: pytest.MonkeyPatch) -> N assert "db down" in failed["err"] -# ── Scheduler wiring (#1967) ─────────────────────────────────────────────────── - - -def test_scheduler_has_trigger_and_dispatch() -> None: - assert hasattr(scheduler, "trigger_geoportal_coords_backfill_run") - loop_src = inspect.getsource(scheduler.scheduler_loop) - assert 'source == "geoportal_coords_backfill"' in loop_src - assert "trigger_geoportal_coords_backfill_run(db, sch)" in loop_src - - -def test_trigger_claims_run_and_runs_in_executor() -> None: - src = inspect.getsource(scheduler.trigger_geoportal_coords_backfill_run) - assert "_claim_run(db, schedule_row)" in src - # sync DB-only task → run_in_executor (not a bare async call). - assert "run_in_executor" in src - assert "run_geoportal_coords_backfill" in src - - -def test_dispatch_runs_before_geocode_missing_listings_in_source_docstring() -> None: - """Header docs must place geoportal_coords_backfill BEFORE geocode_missing_listings. - - Ordering is the whole point (#1967): exact house-level match should claim NULL-coord - listings before the coarser Nominatim/Yandex backfill does. - """ - header = scheduler.__doc__ or "" - geoportal_idx = header.find("geoportal_coords_backfill") - nominatim_idx = header.find("geocode_missing_listings") - assert geoportal_idx != -1 - assert nominatim_idx != -1 - assert geoportal_idx < nominatim_idx - - # ── Migration content (#1967) ────────────────────────────────────────────────── diff --git a/tradein-mvp/backend/tests/tasks/test_cadastral_geo_match.py b/tradein-mvp/backend/tests/tasks/test_cadastral_geo_match.py index c2c7b394..cf6f2257 100644 --- a/tradein-mvp/backend/tests/tasks/test_cadastral_geo_match.py +++ b/tradein-mvp/backend/tests/tasks/test_cadastral_geo_match.py @@ -6,7 +6,6 @@ emitted SQL via the text() clauses and inspect.getsource and check: - the LATERAL KNN shape (ST_DWithin geography gate + `<->` KNN order + LIMIT 1), - the threshold/only_missing params are bound (psycopg-v3 CAST discipline, no :p::type), - the refresh does a single bulk FDW scan (TRUNCATE + INSERT ... FROM gendesign_cad_buildings), - - scheduler wiring (trigger fn + dispatch branch), - migration 124 (table + GIST) and 125 (schedule seed) contents. Plus a fake-db behavioural test driving the chunk-loop + counter logic without Postgres. @@ -30,7 +29,6 @@ import pytest # settings needs a DSN at import (same dance as the sibling tests); these are static/fake-db. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") -from app.services import scheduler from app.tasks import cadastral_geo_match as cgm _SQL_DIR = Path(__file__).resolve().parents[2] / "data" / "sql" @@ -146,24 +144,6 @@ def test_matcher_docstring_marks_approximation_and_threshold() -> None: assert "threshold_m=%d" in _MATCH_SRC -# ── Scheduler wiring ────────────────────────────────────────────────────────── - - -def test_scheduler_has_trigger_and_dispatch() -> None: - assert hasattr(scheduler, "trigger_cadastral_geo_match_run") - loop_src = inspect.getsource(scheduler.scheduler_loop) - assert 'source == "cadastral_geo_match"' in loop_src - assert "trigger_cadastral_geo_match_run(db, sch)" in loop_src - - -def test_trigger_claims_run_and_runs_in_executor() -> None: - src = inspect.getsource(scheduler.trigger_cadastral_geo_match_run) - assert "_claim_run(db, schedule_row)" in src - # sync DB-only task → run_in_executor (not a bare async call). - assert "run_in_executor" in src - assert "run_cadastral_geo_match" in src - - # ── Migration content ───────────────────────────────────────────────────────── diff --git a/tradein-mvp/backend/tests/tasks/test_newbuilding_enrich_backfill.py b/tradein-mvp/backend/tests/tasks/test_newbuilding_enrich_backfill.py index 44852cce..452ffc3b 100644 --- a/tradein-mvp/backend/tests/tasks/test_newbuilding_enrich_backfill.py +++ b/tradein-mvp/backend/tests/tasks/test_newbuilding_enrich_backfill.py @@ -521,15 +521,12 @@ async def test_backfill_resolve_then_idempotent_rerun_no_reresolve() -> None: # --------------------------------------------------------------------------- -# #973 (950-E2): nightly schedule — run_newbuilding_enrich wrapper, scheduler -# trigger/dispatch wiring, and the seed migration 103. +# #973 (950-E2): nightly schedule — run_newbuilding_enrich wrapper + seed migration 103. # --------------------------------------------------------------------------- -import inspect # noqa: E402 import re # noqa: E402 from pathlib import Path # noqa: E402 -from app.services import scheduler # noqa: E402 from app.tasks import newbuilding_enrich_backfill as task_mod # noqa: E402 _SQL_DIR = Path(__file__).resolve().parents[2] / "data" / "sql" @@ -616,71 +613,6 @@ async def test_run_wrapper_marks_failed_and_reraises(monkeypatch: pytest.MonkeyP assert "cian exploded" in failed["err"] -# ── Scheduler wiring: trigger fn claims + dispatches; skip when already running ── - - -def test_trigger_fn_exists_and_is_async() -> None: - fn = getattr(scheduler, "trigger_newbuilding_enrich_run", None) - assert fn is not None, "trigger_newbuilding_enrich_run missing from scheduler" - assert inspect.iscoroutinefunction(fn) - - -def test_dispatch_branch_wired() -> None: - """scheduler_loop dispatches source='newbuilding_enrich' to the trigger.""" - loop_src = inspect.getsource(scheduler.scheduler_loop) - assert 'source == "newbuilding_enrich"' in loop_src - assert "trigger_newbuilding_enrich_run(db, sch)" in loop_src - - -def test_trigger_claims_run_and_delegates_to_task() -> None: - """Trigger claims via _claim_run and delegates to run_newbuilding_enrich (async task).""" - trig_src = inspect.getsource(scheduler.trigger_newbuilding_enrich_run) - assert "_claim_run(db, schedule_row)" in trig_src - assert "run_newbuilding_enrich" in trig_src - # #1182 P2: detached spawn централизован в _spawn_tracked (внутри — asyncio.create_task - # + registry strong-ref для graceful SIGTERM-drain). - assert "_spawn_tracked(_run())" in trig_src - - -@pytest.mark.asyncio -async def test_trigger_claims_and_dispatches(monkeypatch: pytest.MonkeyPatch) -> None: - """Happy path: _claim_run returns a run_id → an asyncio task is spawned.""" - monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: 4242) - spawned: list = [] - real_create_task = scheduler.asyncio.create_task - - def _spy_create_task(coro): - spawned.append(coro) - # Wrap the real coro so it actually runs (and closes the SessionLocal). - return real_create_task(coro) - - monkeypatch.setattr(scheduler.asyncio, "create_task", _spy_create_task) - # Stop the spawned _run() from touching a real DB / importing the heavy task. - monkeypatch.setattr(scheduler, "SessionLocal", lambda: MagicMock()) - - run_id = await scheduler.trigger_newbuilding_enrich_run( - MagicMock(), {"source": "newbuilding_enrich", "default_params": {}} - ) - - assert run_id == 4242 - assert len(spawned) == 1 - - -@pytest.mark.asyncio -async def test_trigger_skips_when_already_running(monkeypatch: pytest.MonkeyPatch) -> None: - """_claim_run None (already running / lock held) → trigger returns None, no task.""" - monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: None) - spawned: list = [] - monkeypatch.setattr(scheduler.asyncio, "create_task", lambda coro: spawned.append(coro)) - - run_id = await scheduler.trigger_newbuilding_enrich_run( - MagicMock(), {"source": "newbuilding_enrich", "default_params": {}} - ) - - assert run_id is None - assert spawned == [] - - # ── Migration 103 ───────────────────────────────────────────────────────────── diff --git a/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py b/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py index 3b32277a..43ec5a5b 100644 --- a/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py +++ b/tradein-mvp/backend/tests/test_asking_to_sold_ratio.py @@ -4,8 +4,7 @@ recompute_asking_to_sold_ratios is SQL-heavy, so most assertions are static: we emitted SQL via .text and check the TRUE-MIRROR shape (DELETE WHERE district='' BEFORE the re-derivation INSERT), that the derivation reuses the exact 080 logic (deals + listings, the trailing-12mo window, the 30/30 threshold, the global -1 fallback row), and the psycopg-v3 -cast discipline (no :param::type). We also assert the scheduler wiring (trigger fn + dispatch -branch) and the migration 082 contents. +cast discipline (no :param::type). We also assert the migration 082 contents. Plus one cheap behavioural test: a fake db (monkeypatched .execute) drives the counter logic without a real Postgres. @@ -26,7 +25,6 @@ import pytest # static / fake-db; no live database is touched. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") -from app.services import scheduler from app.tasks import asking_to_sold_ratio as ratio_mod _SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql" @@ -196,29 +194,6 @@ def test_task_returns_counters_and_finalises_run() -> None: assert "raise" in _TASK_SRC # re-raise on failure — no silent swallow -# ── Scheduler wiring ────────────────────────────────────────────────────────── - - -def test_trigger_fn_exists_and_is_async() -> None: - fn = getattr(scheduler, "trigger_asking_to_sold_ratio_run", None) - assert fn is not None, "trigger_asking_to_sold_ratio_run missing from scheduler" - assert inspect.iscoroutinefunction(fn) - - -def test_dispatch_branch_wired() -> None: - """scheduler_loop dispatches source='asking_to_sold_ratio_refresh' to the trigger.""" - loop_src = inspect.getsource(scheduler.scheduler_loop) - assert 'source == "asking_to_sold_ratio_refresh"' in loop_src - assert "trigger_asking_to_sold_ratio_run(db, sch)" in loop_src - - -def test_trigger_runs_sync_task_in_executor() -> None: - """Sync DB-only task → run_in_executor (mirror snapshot), not a bare await.""" - trig_src = inspect.getsource(scheduler.trigger_asking_to_sold_ratio_run) - assert "run_in_executor(None, recompute_asking_to_sold_ratios" in trig_src - assert "_claim_run(db, schedule_row)" in trig_src - - # ── Migration 082 ───────────────────────────────────────────────────────────── diff --git a/tradein-mvp/backend/tests/test_deactivate_stale_avito.py b/tradein-mvp/backend/tests/test_deactivate_stale_avito.py index b9ae6189..9dffac4c 100644 --- a/tradein-mvp/backend/tests/test_deactivate_stale_avito.py +++ b/tradein-mvp/backend/tests/test_deactivate_stale_avito.py @@ -1,8 +1,8 @@ """Tests for the stale avito listing deactivation task (#759). -Static assertions (SQL shape, psycopg-v3 cast discipline, scheduler wiring) mirror -the pattern established in test_listing_source_snapshot.py. One behavioural test drives -the counter logic via a fake db without a live database. +Static assertions (SQL shape, psycopg-v3 cast discipline) mirror the pattern established +in test_listing_source_snapshot.py. One behavioural test drives the counter logic via a +fake db without a live database. """ import inspect @@ -13,11 +13,10 @@ from typing import Any import pytest -# Stub DATABASE_URL before any app import (same guard as test_scheduler.py / -# test_listing_source_snapshot.py — these tests are static/fake-db, no live DB). +# Stub DATABASE_URL before any app import (same guard as test_listing_source_snapshot.py — +# these tests are static/fake-db, no live DB). os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") -from app.services import scheduler from app.tasks import deactivate_stale_avito as task_mod _SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql" @@ -103,31 +102,6 @@ def test_task_commits_and_rollbacks_on_error() -> None: assert "db.rollback()" in _GENERIC_TASK_SRC -# ── Scheduler wiring ────────────────────────────────────────────────────────── - - -def test_trigger_fn_exists_and_is_async() -> None: - fn = getattr(scheduler, "trigger_deactivate_stale_avito_run", None) - assert fn is not None, "trigger_deactivate_stale_avito_run missing from scheduler" - assert inspect.iscoroutinefunction(fn) - - -def test_dispatch_branch_wired() -> None: - """scheduler_loop dispatches deactivate_stale_* sources to the generic trigger.""" - loop_src = inspect.getsource(scheduler.scheduler_loop) - assert 'source.startswith("deactivate_stale_")' in loop_src - assert "trigger_deactivate_stale_run(db, sch)" in loop_src - - -def test_trigger_runs_sync_task_in_executor() -> None: - """Sync DB-only task → run_in_executor, not a bare await.""" - trig_src = inspect.getsource(scheduler.trigger_deactivate_stale_avito_run) - # run_in_executor call may span two lines; check both parts separately. - assert "run_in_executor(" in trig_src - assert "deactivate_stale_avito_listings" in trig_src - assert "_claim_run(db, schedule_row)" in trig_src - - # ── Migration 090 ───────────────────────────────────────────────────────────── diff --git a/tradein-mvp/backend/tests/test_deactivate_stale_listings.py b/tradein-mvp/backend/tests/test_deactivate_stale_listings.py index 1018896a..f41dc200 100644 --- a/tradein-mvp/backend/tests/test_deactivate_stale_listings.py +++ b/tradein-mvp/backend/tests/test_deactivate_stale_listings.py @@ -10,7 +10,6 @@ import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") -from app.services import scheduler from app.tasks import deactivate_stale_avito as task_mod _SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql" @@ -77,32 +76,6 @@ def test_compat_wrapper_calls_generic_with_avito_defaults() -> None: assert "segments=None" in _COMPAT_FN_SRC -def test_trigger_deactivate_stale_run_exists_and_is_async() -> None: - fn = getattr(scheduler, "trigger_deactivate_stale_run", None) - assert fn is not None, "trigger_deactivate_stale_run missing from scheduler" - assert inspect.iscoroutinefunction(fn) - - -def test_dispatch_branch_uses_startswith() -> None: - loop_src = inspect.getsource(scheduler.scheduler_loop) - assert 'source.startswith("deactivate_stale_")' in loop_src - assert "trigger_deactivate_stale_run(db, sch)" in loop_src - - -def test_trigger_runs_sync_task_in_executor() -> None: - trig_src = inspect.getsource(scheduler.trigger_deactivate_stale_run) - assert "run_in_executor(" in trig_src - assert "deactivate_stale_listings" in trig_src - assert "_claim_run(db, schedule_row)" in trig_src - - -def test_trigger_reads_params_from_default_params() -> None: - trig_src = inspect.getsource(scheduler.trigger_deactivate_stale_run) - assert "listing_source" in trig_src - assert "ttl_days" in trig_src - assert "segments" in trig_src - - class _FakeResult: def __init__(self, rowcount: int) -> None: self.rowcount = rowcount diff --git a/tradein-mvp/backend/tests/test_deals_freshness_monitor.py b/tradein-mvp/backend/tests/test_deals_freshness_monitor.py index f725e282..e95ef115 100644 --- a/tradein-mvp/backend/tests/test_deals_freshness_monitor.py +++ b/tradein-mvp/backend/tests/test_deals_freshness_monitor.py @@ -8,12 +8,11 @@ - кастомный lag_allowance_days. 2. check_deals_freshness с FakeDB (no-alert / alert / empty→mark_failed). 3. Свойства миграции 162 (по образцу test_deactivate_stale_listings). - 4. Регистрацию в scheduler-dispatch + kit product_handlers (registry остаётся зелёным). + 4. Регистрацию в kit product_handlers (registry остаётся зелёным). """ from __future__ import annotations -import inspect import os import re from datetime import UTC, date, datetime @@ -24,7 +23,6 @@ import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") -from app.services import scheduler from app.services.product_handlers import build_product_handlers from app.tasks import deals_freshness_monitor as mon @@ -223,26 +221,7 @@ def test_migration_162_no_psycopg_trap() -> None: assert not re.search(r":\w+::", sql) -# ── Регистрация в scheduler + kit registry ──────────────────────────────────── - - -def test_trigger_exists_and_is_async() -> None: - fn = getattr(scheduler, "trigger_deals_freshness_monitor_run", None) - assert fn is not None, "trigger_deals_freshness_monitor_run missing from scheduler" - assert inspect.iscoroutinefunction(fn) - - -def test_dispatch_branch_present() -> None: - loop_src = inspect.getsource(scheduler.scheduler_loop) - assert 'source == "deals_freshness_monitor"' in loop_src - assert "trigger_deals_freshness_monitor_run(db, sch)" in loop_src - - -def test_trigger_runs_sync_task_in_executor() -> None: - trig_src = inspect.getsource(scheduler.trigger_deals_freshness_monitor_run) - assert "run_in_executor(" in trig_src - assert "check_deals_freshness" in trig_src - assert "_claim_run(db, schedule_row)" in trig_src +# ── Регистрация в kit registry ───────────────────────────────────────────────── def test_kit_product_handler_registered() -> None: diff --git a/tradein-mvp/backend/tests/test_house_dedup_merge.py b/tradein-mvp/backend/tests/test_house_dedup_merge.py index a8aac92a..c6bacef4 100644 --- a/tradein-mvp/backend/tests/test_house_dedup_merge.py +++ b/tradein-mvp/backend/tests/test_house_dedup_merge.py @@ -26,7 +26,6 @@ import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from app.services import house_dedup_merge as hdm -from app.services import scheduler _SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql" _MIGRATION_135 = _SQL_DIR / "135_scrape_schedules_seed_house_dedup_merge.sql" @@ -560,30 +559,6 @@ def test_run_wrapper_marks_failed_on_error(monkeypatch: pytest.MonkeyPatch) -> N assert "merge exploded" in failed["err"] -# ── Scheduler wiring ────────────────────────────────────────────────────────── - - -def test_scheduler_has_trigger_and_dispatch() -> None: - assert hasattr(scheduler, "trigger_house_dedup_merge_run") - loop_src = inspect.getsource(scheduler.scheduler_loop) - assert 'source == "house_dedup_merge"' in loop_src - assert "trigger_house_dedup_merge_run(db, sch)" in loop_src - - -def test_trigger_claims_run_and_runs_in_executor() -> None: - src = inspect.getsource(scheduler.trigger_house_dedup_merge_run) - assert "_claim_run(db, schedule_row)" in src - assert "if run_id is None:" in src - # sync DB-only task → run_in_executor (mirror cadastral_geo_match, not a bare await). - assert "run_in_executor" in src - assert "run_house_dedup_merge" in src - # fresh session inside the spawned task + #1182 P2 detached spawn via _spawn_tracked - # (registry strong-ref = RUF006 keep-alive + graceful SIGTERM-drain). - assert "run_db = SessionLocal()" in src - assert "_spawn_tracked(_run())" in src - assert "run_db.close()" in src - - # ── Migration content ───────────────────────────────────────────────────────── diff --git a/tradein-mvp/backend/tests/test_house_imv_backfill_scheduler.py b/tradein-mvp/backend/tests/test_house_imv_backfill_scheduler.py deleted file mode 100644 index 8bd52594..00000000 --- a/tradein-mvp/backend/tests/test_house_imv_backfill_scheduler.py +++ /dev/null @@ -1,237 +0,0 @@ -"""Tests for wiring house_imv_backfill as an in-app scheduler source (#854). - -The bulk IMV backfill service (services/house_imv_backfill.backfill_house_imv) is -already built + covered elsewhere; this gate covers ONLY the scheduler wiring added -by #854: - - trigger_house_imv_backfill_run shape (_claim_run → asyncio.create_task → fresh - SessionLocal → mark_done/mark_failed lifecycle → add_done_callback / RUF006), - - dispatch branch in scheduler_loop routes source=="house_imv_backfill", - - migration 132 seeds an idempotent scrape_schedules row with default_params. - -Convention mirrors tests/tasks/test_cadastral_geo_match.py: hermetic static analysis -(inspect.getsource + migration file read) plus a fake-async behavioural test driving -the trigger's lifecycle without a live Postgres. -""" - -from __future__ import annotations - -import asyncio -import inspect -import os -import re -from pathlib import Path -from typing import Any - -import pytest - -# settings needs a DSN at import (same dance as the sibling tests); these are static/fake-db. -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") - -from app.services import scheduler - -_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql" -_MIGRATION_132 = _SQL_DIR / "132_scrape_schedules_seed_house_imv.sql" - -_TRIGGER_SRC = inspect.getsource(scheduler.trigger_house_imv_backfill_run) - - -# ── Scheduler wiring ────────────────────────────────────────────────────────── - - -def test_scheduler_has_trigger_and_dispatch() -> None: - assert hasattr(scheduler, "trigger_house_imv_backfill_run") - loop_src = inspect.getsource(scheduler.scheduler_loop) - assert 'source == "house_imv_backfill"' in loop_src - assert "trigger_house_imv_backfill_run(db, sch)" in loop_src - - -def test_trigger_mirrors_backfill_sibling_pattern() -> None: - """_claim_run gate → fresh SessionLocal → backfill_house_imv → add_done_callback.""" - assert "_claim_run(db, schedule_row)" in _TRIGGER_SRC - assert "if run_id is None:" in _TRIGGER_SRC - # fresh session inside the spawned task (not the scheduler-tick session). - assert "run_db = SessionLocal()" in _TRIGGER_SRC - assert "backfill_house_imv" in _TRIGGER_SRC - # async service → bare await (NOT run_in_executor, unlike the sync DB-only triggers). - assert "run_in_executor" not in _TRIGGER_SRC - # #1182 P2: detached spawn через _spawn_tracked — strong-ref в registry (RUF006 + - # graceful SIGTERM-drain), done-callback ретривит exception. Заменил прежний - # `task = asyncio.create_task(_run()); task.add_done_callback(...)`. - assert "_spawn_tracked(_run())" in _TRIGGER_SRC - assert "run_db.close()" in _TRIGGER_SRC - - -def test_trigger_owns_run_lifecycle() -> None: - """No tasks/*-wrapper for this service → the trigger finalises scrape_runs itself.""" - assert "runs_mod.mark_done(" in _TRIGGER_SRC - assert "runs_mod.mark_failed(" in _TRIGGER_SRC - assert "logger.exception" in _TRIGGER_SRC - # heartbeat callback keeps the long IMV run alive vs reap_zombies (#1363). - assert "heartbeat=" in _TRIGGER_SRC - assert "update_heartbeat" in _TRIGGER_SRC - - -def test_trigger_reads_default_params() -> None: - """default_params drive batch_size / request_delay_sec / only_status with safe defaults.""" - assert 'params.get("batch_size"' in _TRIGGER_SRC - assert 'params.get("request_delay_sec"' in _TRIGGER_SRC - assert 'params.get("only_status"' in _TRIGGER_SRC - - -# ── Migration content ───────────────────────────────────────────────────────── - - -def test_migration_132_seeds_schedule_idempotent() -> None: - sql = _MIGRATION_132.read_text(encoding="utf-8") - assert "INSERT INTO scrape_schedules" in sql - assert "'house_imv_backfill'" in sql - assert "ON CONFLICT (source) DO NOTHING" in sql - assert "BEGIN;" in sql and "COMMIT;" in sql - - -def test_migration_132_next_run_not_null_tomorrow() -> None: - """next_run_at NOT NULL = tomorrow (avoid deploy-time fire), per seed convention.""" - sql = _MIGRATION_132.read_text(encoding="utf-8") - assert "CURRENT_DATE + INTERVAL '1 day'" in sql - assert "make_interval(hours => 15)" in sql - - -def test_migration_132_default_params() -> None: - sql = _MIGRATION_132.read_text(encoding="utf-8") - assert '"batch_size": 50' in sql - assert '"request_delay_sec": 5' in sql - assert '"only_status": "pending"' in sql - - -def test_migration_132_no_psycopg_v3_colon_colon_cast() -> None: - """psycopg v3: never :param::type — must use CAST(:param AS type).""" - sql = _MIGRATION_132.read_text(encoding="utf-8") - assert not re.search(r":\w+::", sql) - - -def test_migration_132_matches_seed_column_set() -> None: - """Exact column set of existing scrape_schedules seeds (source..default_params).""" - sql = _MIGRATION_132.read_text(encoding="utf-8") - for col in ( - "source", - "enabled", - "window_start_hour", - "window_end_hour", - "next_run_at", - "default_params", - ): - assert col in sql - - -# ── Fake-async behavioural: trigger lifecycle without Postgres ──────────────── - - -class _FakeDB: - """Minimal Session stand-in: only needs to be a distinct object + close().""" - - def __init__(self) -> None: - self.closed = 0 - - def close(self) -> None: - self.closed += 1 - - -def test_trigger_skips_when_claim_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: - """If _claim_run finds a running run (None) → trigger returns None, spawns nothing.""" - monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: None) - - async def _go() -> int | None: - return await scheduler.trigger_house_imv_backfill_run( - _FakeDB(), # type: ignore[arg-type] - {"source": "house_imv_backfill", "default_params": {}}, - ) - - assert asyncio.run(_go()) is None - - -def test_trigger_claims_runs_backfill_and_marks_done(monkeypatch: pytest.MonkeyPatch) -> None: - """Happy path: claim run → backfill_house_imv awaited with params → mark_done(counters).""" - captured: dict[str, Any] = {} - marked: dict[str, Any] = {} - - monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: 77) - monkeypatch.setattr(scheduler, "SessionLocal", _FakeDB) - monkeypatch.setattr(scheduler.runs_mod, "update_heartbeat", lambda *a, **k: None) - monkeypatch.setattr( - scheduler.runs_mod, - "mark_done", - lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)), - ) - monkeypatch.setattr(scheduler.runs_mod, "mark_failed", lambda *a, **k: None) - - class _Result: - checked, saved, skipped, errors, duration_sec = 10, 6, 2, 2, 42.0 - - async def _fake_backfill(_db: Any, **kwargs: Any) -> _Result: - captured.update(kwargs) - return _Result() - - import app.services.house_imv_backfill as _svc - - monkeypatch.setattr(_svc, "backfill_house_imv", _fake_backfill) - - async def _go() -> int | None: - run_id = await scheduler.trigger_house_imv_backfill_run( - _FakeDB(), # type: ignore[arg-type] - { - "source": "house_imv_backfill", - "default_params": { - "batch_size": 25, - "request_delay_sec": 3, - "only_status": "transient_error", - }, - }, - ) - # let the spawned _run() task finish. - await asyncio.sleep(0) - await asyncio.sleep(0) - return run_id - - run_id = asyncio.run(_go()) - assert run_id == 77 - assert captured["batch_size"] == 25 - assert captured["request_delay_sec"] == 3.0 - assert captured["only_status"] == "transient_error" - assert callable(captured["heartbeat"]) - assert marked["run_id"] == 77 - assert marked["counters"]["checked"] == 10 - assert marked["counters"]["saved"] == 6 - - -def test_trigger_marks_failed_on_backfill_crash(monkeypatch: pytest.MonkeyPatch) -> None: - """If backfill_house_imv raises → trigger marks the run failed (does not bubble).""" - failed: dict[str, Any] = {} - - monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: 88) - monkeypatch.setattr(scheduler, "SessionLocal", _FakeDB) - monkeypatch.setattr(scheduler.runs_mod, "update_heartbeat", lambda *a, **k: None) - monkeypatch.setattr(scheduler.runs_mod, "mark_done", lambda *a, **k: None) - monkeypatch.setattr( - scheduler.runs_mod, - "mark_failed", - lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err), - ) - - async def _boom(_db: Any, **kwargs: Any) -> None: - raise RuntimeError("imv 403") - - import app.services.house_imv_backfill as _svc - - monkeypatch.setattr(_svc, "backfill_house_imv", _boom) - - async def _go() -> None: - await scheduler.trigger_house_imv_backfill_run( - _FakeDB(), # type: ignore[arg-type] - {"source": "house_imv_backfill", "default_params": {}}, - ) - await asyncio.sleep(0) - await asyncio.sleep(0) - - asyncio.run(_go()) - assert failed["run_id"] == 88 - assert "imv 403" in failed["err"] diff --git a/tradein-mvp/backend/tests/test_kit_registry_completeness.py b/tradein-mvp/backend/tests/test_kit_registry_completeness.py deleted file mode 100644 index fef35c0d..00000000 --- a/tradein-mvp/backend/tests/test_kit_registry_completeness.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Полнота kit-registry vs боевой scheduler-dispatch (#2192). - -Гарантирует ship-dark инвариант: КАЖДЫЙ source, который обрабатывает боевой -`app.services.scheduler.scheduler_loop` (if/elif + startswith), резолвится в -kit-registry (`build_registry(product_handlers=build_product_handlers(ctx))`). -Тест ПАДАЕТ, если source добавили в боевой dispatch, но забыли в registry — -иначе после переключения USE_KIT_SCHEDULER=true такой source молча не запустится. - -Источник истины — сам файл боевого scheduler'а: парсим `source == "X"` (exact) -и `source.startswith("X")` (wildcard), чтобы при дрейфе dispatch тест ловил -рассинхрон автоматически, а не по захардкоженному списку. -""" - -from __future__ import annotations - -import os -import re -from pathlib import Path - -os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") - -from scraper_kit.orchestration.scheduler import ( - SchedulerContext, - build_registry, - resolve_handler, -) - -import app.services.scheduler as _sched_mod -from app.services.product_handlers import build_product_handlers -from app.services.scraper_adapters import ( - RealEnrichmentJobs, - RealMatcherAdapter, - RealScraperConfig, - RealSessionFactory, -) - - -def _production_sources() -> tuple[set[str], set[str]]: - """(exact, prefixes) source'ов из боевого scheduler_loop dispatch.""" - src = Path(_sched_mod.__file__).read_text(encoding="utf-8") - exact = set(re.findall(r'source == "([^"]+)"', src)) - prefixes = set(re.findall(r'source\.startswith\("([^"]+)"\)', src)) - return exact, prefixes - - -def _build_registry() -> dict: - ctx = SchedulerContext( - config=RealScraperConfig(), - matcher=RealMatcherAdapter(), - enrichment=RealEnrichmentJobs(), - session_factory=RealSessionFactory(), - ) - return build_registry(product_handlers=build_product_handlers(ctx)) - - -def test_production_dispatch_parse_sane() -> None: - """Sanity: парсер видит непустой набор source'ов (защита от regex-дрейфа).""" - exact, prefixes = _production_sources() - # На момент #2192 боевой dispatch держит 25 exact + 1 wildcard-семейство. - assert len(exact) >= 25, f"parse regressed — got {len(exact)} exact sources" - assert "deactivate_stale_" in prefixes - - -def test_every_production_source_resolves_in_kit_registry() -> None: - """Каждый боевой exact-source резолвится в kit-registry.""" - registry = _build_registry() - exact, _ = _production_sources() - missing = [s for s in sorted(exact) if resolve_handler(s, registry) is None] - assert not missing, f"sources missing from kit registry: {missing}" - - -def test_wildcard_family_resolves() -> None: - """Каждый deactivate_stale_* член семейства резолвится через wildcard-Handler.""" - registry = _build_registry() - _, prefixes = _production_sources() - for prefix in sorted(prefixes): - for member in ("avito", "yandex", "cian"): - source = f"{prefix}{member}" - assert ( - resolve_handler(source, registry) is not None - ), f"wildcard source {source} did not resolve" - - -def test_unknown_source_does_not_resolve() -> None: - """Негативный контроль: несуществующий source → None (тест реально дискриминирует).""" - registry = _build_registry() - assert resolve_handler("totally_unknown_source_xyz", registry) is None diff --git a/tradein-mvp/backend/tests/test_listing_source_snapshot.py b/tradein-mvp/backend/tests/test_listing_source_snapshot.py index 8b20acf8..1f98a28c 100644 --- a/tradein-mvp/backend/tests/test_listing_source_snapshot.py +++ b/tradein-mvp/backend/tests/test_listing_source_snapshot.py @@ -3,7 +3,7 @@ Writer (snapshot_listing_sources) is SQL-heavy, so most assertions are static: we read the emitted SQL via inspect/text-attr and check the upsert + event-diff CTE shape, table/ column names, and the psycopg-v3 cast discipline (no :param::type). We also assert the -scheduler wiring (trigger fn + dispatch branch) and the migration 079 contents. +migration 079 contents. Plus one cheap behavioural test: a fake db (monkeypatched .execute) drives the counter logic without a real Postgres. @@ -24,7 +24,6 @@ import pytest # static / fake-db; no live database is touched. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") -from app.services import scheduler from app.tasks import listing_source_snapshot as snap_mod _SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql" @@ -133,29 +132,6 @@ def test_writer_returns_counters_and_finalises_run() -> None: assert "mark_failed" in _WRITER_SRC -# ── Scheduler wiring ────────────────────────────────────────────────────────── - - -def test_trigger_fn_exists_and_is_async() -> None: - fn = getattr(scheduler, "trigger_listing_source_snapshot_run", None) - assert fn is not None, "trigger_listing_source_snapshot_run missing from scheduler" - assert inspect.iscoroutinefunction(fn) - - -def test_dispatch_branch_wired() -> None: - """scheduler_loop dispatches source='listing_source_snapshot' to the trigger.""" - loop_src = inspect.getsource(scheduler.scheduler_loop) - assert 'source == "listing_source_snapshot"' in loop_src - assert "trigger_listing_source_snapshot_run(db, sch)" in loop_src - - -def test_trigger_runs_sync_task_in_executor() -> None: - """Sync DB-only task → run_in_executor (mirror rosreestr), not a bare await.""" - trig_src = inspect.getsource(scheduler.trigger_listing_source_snapshot_run) - assert "run_in_executor(None, snapshot_listing_sources" in trig_src - assert "_claim_run(db, schedule_row)" in trig_src - - # ── Migration 079 ───────────────────────────────────────────────────────────── diff --git a/tradein-mvp/backend/tests/test_scheduler.py b/tradein-mvp/backend/tests/test_scheduler.py index 6046f815..e204d3a2 100644 --- a/tradein-mvp/backend/tests/test_scheduler.py +++ b/tradein-mvp/backend/tests/test_scheduler.py @@ -1,29 +1,19 @@ -"""Offline smoke for scheduler logic.""" +"""Offline smoke for compute_next_run_at (app/services/scheduler.py). + +#2397 Part C removed the legacy scheduler_loop + trigger_*/get_due_schedules/_claim_run/ +reap_zombies/_spawn_tracked/_drain_inflight machinery from app.services.scheduler — the kit +scheduler (scraper_kit.orchestration.scheduler, covered by test_scraper_kit_scheduler_parity.py) +is now the only scheduling path. compute_next_run_at survives as a shared helper (still used +by admin.py for the operator-facing "next run" preview), so this file now covers only it. +""" -import asyncio import os -from unittest.mock import AsyncMock, MagicMock os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from datetime import UTC, datetime -import pytest - -import app.services.scheduler as sched -from app.core import shutdown as sd -from app.services.scheduler import ZOMBIE_THRESHOLD_HOURS, compute_next_run_at - - -@pytest.fixture(autouse=True) -def _reset_drain_state() -> None: - """#1182 P2: shutdown-флаг и registry детей — module-global; чистим вокруг каждого - теста (изоляция от detached-задач, что триггеры спавнят в _inflight_tasks).""" - sd.reset_shutdown() - sched._inflight_tasks.clear() - yield - sched._inflight_tasks.clear() - sd.reset_shutdown() +from app.services.scheduler import compute_next_run_at def test_compute_next_run_in_normal_window() -> None: @@ -49,10 +39,6 @@ def test_compute_next_run_window_1_hour() -> None: assert next_at.hour == 3 -def test_zombie_threshold_constant() -> None: - assert ZOMBIE_THRESHOLD_HOURS > 0 - - def test_compute_next_run_is_future() -> None: """next_run_at всегда в будущем (завтра) — не в прошлом.""" now = datetime(2026, 5, 23, 3, 30, tzinfo=UTC) @@ -101,725 +87,3 @@ def test_compute_next_run_interval_days_cross_midnight_weekly() -> None: next_at = compute_next_run_at(22, 3, now=now, interval_days=7) assert next_at > now assert next_at.hour >= 22 or next_at.hour < 3 - - -# ── #750: атомарный _claim_run (advisory lock — anti-double-sweep) ────────── - -from collections import deque # noqa: E402 - -from app.services import scheduler as _sched # noqa: E402 - - -class _Scalar: - def __init__(self, v: object) -> None: - self._v = v - - def scalar(self) -> object: - return self._v - - -class _Fetchone: - def __init__(self, row: object) -> None: - self._row = row - - def fetchone(self) -> object: - return self._row - - -class _FakeSchedulerDB: - """Stateful fake моделирующий advisory-lock claim (#750), без реальной БД. - - - SELECT pg_try_advisory_xact_lock → lock_available. - - SELECT 1 FROM scrape_runs (has_running_run) → из running_results (если задано), - иначе по флагу self.running[source]. - - UPDATE scrape_schedules → no-op. - create_run (монкипатчится) выставляет self.running[source]=True. - """ - - def __init__(self, *, lock_available: bool = True, running_results: list[bool] | None = None): - self.lock_available = lock_available - self.running: dict[str, bool] = {} - self._running_results: deque[bool] = deque(running_results or []) - self.commits = 0 - self.rollbacks = 0 - - def execute(self, stmt: object, params: dict | None = None): - sql = str(stmt) - if "pg_try_advisory_xact_lock" in sql: - return _Scalar(self.lock_available) - if "SELECT 1 FROM scrape_runs" in sql: - if self._running_results: - return _Fetchone(object() if self._running_results.popleft() else None) - src = (params or {}).get("source") - return _Fetchone(object() if self.running.get(src) else None) - return _Fetchone(None) # UPDATE scrape_schedules etc. - - def commit(self) -> None: - self.commits += 1 - - def rollback(self) -> None: - self.rollbacks += 1 - - -_SCHED_ROW = { - "source": "avito_city_sweep", - "window_start_hour": 2, - "window_end_hour": 5, - "default_params": {}, -} - - -def test_claim_run_double_call_creates_one_run(monkeypatch: pytest.MonkeyPatch) -> None: - """#750 DoD: два последовательных _claim_run одного source (нет running) → - первый run_id, второй None (create_run вызван РОВНО один раз).""" - db = _FakeSchedulerDB() - calls = {"n": 0} - - def fake_create_run(_d, *, source, params): - calls["n"] += 1 - db.running[source] = True - return 100 + calls["n"] - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - rid1 = _sched._claim_run(db, _SCHED_ROW) - rid2 = _sched._claim_run(db, _SCHED_ROW) - - assert rid1 == 101 - assert rid2 is None - assert calls["n"] == 1 # второй НЕ создал run - - -def test_claim_run_skips_when_advisory_lock_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: - """#750: lock занят конкурентным тиком → return None, create_run НЕ вызван.""" - db = _FakeSchedulerDB(lock_available=False) - calls = {"n": 0} - - def fake_create_run(_d, *, source, params): - calls["n"] += 1 - return 999 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - assert _sched._claim_run(db, _SCHED_ROW) is None - assert calls["n"] == 0 - - -def test_claim_run_rechecks_running_under_lock(monkeypatch: pytest.MonkeyPatch) -> None: - """#750: running появился МЕЖДУ pre-check и взятием лока (double-checked) → - return None + rollback (освобождение лока), create_run НЕ вызван.""" - # pre-check → False (нет running), re-check под локом → True (появился). - db = _FakeSchedulerDB(lock_available=True, running_results=[False, True]) - calls = {"n": 0} - - def fake_create_run(_d, *, source, params): - calls["n"] += 1 - return 999 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - assert _sched._claim_run(db, _SCHED_ROW) is None - assert calls["n"] == 0 - assert db.rollbacks == 1 # лок освобождён - - -# ── #974: trigger_yandex_newbuilding_sweep_run + dispatch ──────────────────── - - -_YANDEX_NB_SWEEP_ROW = { - "source": "yandex_newbuilding_sweep", - "window_start_hour": 2, - "window_end_hour": 5, - "default_params": { - "limit": 5, - "request_delay_sec": 8.0, - "city": "ekaterinburg", - }, -} - - -@pytest.mark.asyncio -async def test_trigger_yandex_newbuilding_sweep_creates_run( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """trigger_yandex_newbuilding_sweep_run claims run + launches create_task (#974).""" - db = _FakeSchedulerDB() - calls: dict[str, list] = {"create_run": [], "sweep": []} - - def fake_create_run(_d, *, source, params): - calls["create_run"].append(source) - db.running[source] = True - return 301 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - # Patch enrich_yandex_newbuilding_sweep to a coroutine that records call - sweep_result = MagicMock() - sweep_result.to_dict.return_value = {} - - async def fake_sweep(run_db, *, limit, **kwargs): - calls["sweep"].append(limit) - return sweep_result - - # Patch inside the _run closure (lazy import) - monkeypatch.setattr( - "app.tasks.yandex_newbuilding_sweep.enrich_yandex_newbuilding_sweep", - fake_sweep, - ) - monkeypatch.setattr(_sched, "SessionLocal", lambda: db) - - # mark_done is called inside _run; mock it so it doesn't fail on fake db - monkeypatch.setattr(_sched.runs_mod, "mark_done", lambda *a, **kw: None) - monkeypatch.setattr(_sched.runs_mod, "mark_failed", lambda *a, **kw: None) - - run_id = await _sched.trigger_yandex_newbuilding_sweep_run(db, _YANDEX_NB_SWEEP_ROW) - - assert run_id == 301 - assert calls["create_run"] == ["yandex_newbuilding_sweep"] - # Drain event-loop so the asyncio.create_task runs - await asyncio.sleep(0) - assert calls["sweep"] == [5] - - -@pytest.mark.asyncio -async def test_trigger_yandex_newbuilding_sweep_skips_when_running( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """trigger_yandex_newbuilding_sweep_run returns None when source already running.""" - db = _FakeSchedulerDB() - db.running["yandex_newbuilding_sweep"] = True - - calls: dict[str, int] = {"n": 0} - - def fake_create_run(_d, *, source, params): - calls["n"] += 1 - return 999 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - result = await _sched.trigger_yandex_newbuilding_sweep_run(db, _YANDEX_NB_SWEEP_ROW) - - assert result is None - assert calls["n"] == 0 - - -@pytest.mark.asyncio -async def test_scheduler_dispatch_routes_yandex_newbuilding_sweep( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Dispatch block в scheduler_loop роутит yandex_newbuilding_sweep → trigger.""" - triggered: list[str] = [] - - async def fake_trigger(db, sch): - triggered.append(sch["source"]) - return 1 - - monkeypatch.setattr(_sched, "trigger_yandex_newbuilding_sweep_run", fake_trigger) - - db = object() - sch = _YANDEX_NB_SWEEP_ROW - source = sch["source"] - # Replicate the elif chain from scheduler_loop for yandex_newbuilding_sweep - if source == "yandex_newbuilding_sweep": - await _sched.trigger_yandex_newbuilding_sweep_run(db, sch) - - assert "yandex_newbuilding_sweep" in triggered - - -# ── #973: trigger_cian_city_sweep_run + dispatch ───────────────────────────── - - -_CIAN_SWEEP_ROW = { - "source": "cian_city_sweep", - "window_start_hour": 2, - "window_end_hour": 5, - "default_params": { - "pages_per_anchor": 3, - "request_delay_sec": 5.0, - "radius_m": 1500, - "detail_top_n": 10, - "enrich_houses": True, - }, -} - - -@pytest.mark.asyncio -async def test_trigger_cian_city_sweep_creates_run(monkeypatch: pytest.MonkeyPatch) -> None: - """trigger_cian_city_sweep_run claims run + launches create_task (#973).""" - db = _FakeSchedulerDB() - calls: dict[str, list] = {"create_run": [], "sweep": []} - - def fake_create_run(_d, *, source, params): - calls["create_run"].append(source) - db.running[source] = True - return 201 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - # Patch run_cian_city_sweep to a coroutine that records call - async def fake_sweep(run_db, *, run_id, **kwargs): - calls["sweep"].append(run_id) - - monkeypatch.setattr(_sched, "run_cian_city_sweep", fake_sweep) - # Patch SessionLocal to return the same db - monkeypatch.setattr(_sched, "SessionLocal", lambda: db) - - run_id = await _sched.trigger_cian_city_sweep_run(db, _CIAN_SWEEP_ROW) - - assert run_id == 201 - assert calls["create_run"] == ["cian_city_sweep"] - # Drain event-loop so the asyncio.create_task runs - await asyncio.sleep(0) - assert calls["sweep"] == [201] - - -@pytest.mark.asyncio -async def test_trigger_cian_city_sweep_skips_when_running(monkeypatch: pytest.MonkeyPatch) -> None: - """trigger_cian_city_sweep_run returns None when source already running.""" - db = _FakeSchedulerDB() - db.running["cian_city_sweep"] = True - - calls: dict[str, int] = {"n": 0} - - def fake_create_run(_d, *, source, params): - calls["n"] += 1 - return 999 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - result = await _sched.trigger_cian_city_sweep_run(db, _CIAN_SWEEP_ROW) - - assert result is None - assert calls["n"] == 0 - - -@pytest.mark.asyncio -async def test_scheduler_dispatch_routes_cian_city_sweep(monkeypatch: pytest.MonkeyPatch) -> None: - """Dispatch block в scheduler_loop роутит cian_city_sweep → trigger_cian_city_sweep_run.""" - triggered: list[str] = [] - - async def fake_trigger(db, sch): - triggered.append(sch["source"]) - return 1 - - monkeypatch.setattr(_sched, "trigger_cian_city_sweep_run", fake_trigger) - - # Simulate the dispatch block for one due schedule - db = object() - sch = _CIAN_SWEEP_ROW - source = sch["source"] - # Replicate the elif chain from scheduler_loop for cian_city_sweep - if source == "cian_city_sweep": - await _sched.trigger_cian_city_sweep_run(db, sch) - - assert "cian_city_sweep" in triggered - - -# ── cian_full_load: trigger_cian_full_load_run + dispatch ──────────────────── - - -_CIAN_FULL_LOAD_ROW = { - "source": "cian_full_load", - "window_start_hour": 20, - "window_end_hour": 22, - "default_params": { - "price_cap_per_bucket": 1400, - "concurrency": 5, - "request_delay_sec": 4.0, - "enrich_detail": False, - "detail_top_n": 0, - }, -} - - -@pytest.mark.asyncio -async def test_trigger_cian_full_load_creates_run(monkeypatch: pytest.MonkeyPatch) -> None: - """trigger_cian_full_load_run claims run + launches run_cian_full_load via create_task.""" - db = _FakeSchedulerDB() - calls: dict[str, list] = {"create_run": [], "load": []} - - def fake_create_run(_d, *, source, params): - calls["create_run"].append(source) - db.running[source] = True - return 301 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - async def fake_full_load(run_db, *, run_id, **kwargs): - calls["load"].append((run_id, kwargs)) - - monkeypatch.setattr(_sched, "run_cian_full_load", fake_full_load) - monkeypatch.setattr(_sched, "SessionLocal", lambda: db) - - run_id = await _sched.trigger_cian_full_load_run(db, _CIAN_FULL_LOAD_ROW) - - assert run_id == 301 - assert calls["create_run"] == ["cian_full_load"] - # Drain event-loop so the asyncio.create_task runs - await asyncio.sleep(0) - assert len(calls["load"]) == 1 - loaded_run_id, kwargs = calls["load"][0] - assert loaded_run_id == 301 - # Params propagated from default_params; resume_run_id forced to None - assert kwargs["price_cap_per_bucket"] == 1400 - assert kwargs["concurrency"] == 5 - assert kwargs["request_delay_sec"] == 4.0 - assert kwargs["enrich_detail"] is False - assert kwargs["detail_top_n"] == 0 - assert kwargs["resume_run_id"] is None - - -@pytest.mark.asyncio -async def test_trigger_cian_full_load_skips_when_running(monkeypatch: pytest.MonkeyPatch) -> None: - """trigger_cian_full_load_run returns None when source already running.""" - db = _FakeSchedulerDB() - db.running["cian_full_load"] = True - - calls: dict[str, int] = {"n": 0} - - def fake_create_run(_d, *, source, params): - calls["n"] += 1 - return 999 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - result = await _sched.trigger_cian_full_load_run(db, _CIAN_FULL_LOAD_ROW) - - assert result is None - assert calls["n"] == 0 - - -@pytest.mark.asyncio -async def test_scheduler_dispatch_routes_cian_full_load(monkeypatch: pytest.MonkeyPatch) -> None: - """Dispatch block в scheduler_loop роутит cian_full_load → trigger_cian_full_load_run.""" - triggered: list[str] = [] - - async def fake_trigger(db, sch): - triggered.append(sch["source"]) - return 1 - - monkeypatch.setattr(_sched, "trigger_cian_full_load_run", fake_trigger) - - db = object() - sch = _CIAN_FULL_LOAD_ROW - source = sch["source"] - # Replicate the elif chain from scheduler_loop for cian_full_load - if source == "cian_full_load": - await _sched.trigger_cian_full_load_run(db, sch) - - assert "cian_full_load" in triggered - - -# ── avito_full_load: trigger_avito_full_load_run + dispatch ────────────────── - - -_AVITO_FULL_LOAD_ROW = { - "source": "avito_full_load", - "window_start_hour": 13, - "window_end_hour": 15, - "default_params": { - "price_cap_per_bucket": 1400, - "concurrency": 5, - "request_delay_sec": 7.0, - "secondary_only": True, - }, -} - - -@pytest.mark.asyncio -async def test_trigger_avito_full_load_creates_run(monkeypatch: pytest.MonkeyPatch) -> None: - """trigger_avito_full_load_run claims run + launches run_avito_full_load via create_task.""" - db = _FakeSchedulerDB() - calls: dict[str, list] = {"create_run": [], "load": []} - - def fake_create_run(_d, *, source, params): - calls["create_run"].append(source) - db.running[source] = True - return 401 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - async def fake_full_load(run_db, *, run_id, **kwargs): - calls["load"].append((run_id, kwargs)) - - monkeypatch.setattr(_sched, "run_avito_full_load", fake_full_load) - monkeypatch.setattr(_sched, "SessionLocal", lambda: db) - - run_id = await _sched.trigger_avito_full_load_run(db, _AVITO_FULL_LOAD_ROW) - - assert run_id == 401 - assert calls["create_run"] == ["avito_full_load"] - # Drain event-loop so the asyncio.create_task runs - await asyncio.sleep(0) - assert len(calls["load"]) == 1 - loaded_run_id, kwargs = calls["load"][0] - assert loaded_run_id == 401 - # Params propagated from default_params; resume_run_id forced to None - assert kwargs["price_cap_per_bucket"] == 1400 - assert kwargs["concurrency"] == 5 - assert kwargs["request_delay_sec"] == 7.0 - assert kwargs["secondary_only"] is True - assert kwargs["resume_run_id"] is None - - -@pytest.mark.asyncio -async def test_trigger_avito_full_load_skips_when_running(monkeypatch: pytest.MonkeyPatch) -> None: - """trigger_avito_full_load_run returns None when source already running.""" - db = _FakeSchedulerDB() - db.running["avito_full_load"] = True - - calls: dict[str, int] = {"n": 0} - - def fake_create_run(_d, *, source, params): - calls["n"] += 1 - return 999 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - result = await _sched.trigger_avito_full_load_run(db, _AVITO_FULL_LOAD_ROW) - - assert result is None - assert calls["n"] == 0 - - -@pytest.mark.asyncio -async def test_scheduler_dispatch_routes_avito_full_load(monkeypatch: pytest.MonkeyPatch) -> None: - """Dispatch block в scheduler_loop роутит avito_full_load → trigger_avito_full_load_run.""" - triggered: list[str] = [] - - async def fake_trigger(db, sch): - triggered.append(sch["source"]) - return 1 - - monkeypatch.setattr(_sched, "trigger_avito_full_load_run", fake_trigger) - - db = object() - sch = _AVITO_FULL_LOAD_ROW - source = sch["source"] - # Replicate the elif chain from scheduler_loop for avito_full_load - if source == "avito_city_sweep": - pass - elif source == "avito_full_load": - await _sched.trigger_avito_full_load_run(db, sch) - - assert "avito_full_load" in triggered - - -# ── avito_full_load incremental_days + avito_full_load_exhaustive split ─────── - - -@pytest.mark.asyncio -async def test_trigger_avito_full_load_reads_incremental_days( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """incremental_days из default_params прокидывается в run_avito_full_load.""" - db = _FakeSchedulerDB() - calls: dict[str, list] = {"load": []} - - def fake_create_run(_d, *, source, params): - db.running[source] = True - return 410 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - async def fake_full_load(run_db, *, run_id, **kwargs): - calls["load"].append(kwargs) - - monkeypatch.setattr(_sched, "run_avito_full_load", fake_full_load) - monkeypatch.setattr(_sched, "SessionLocal", lambda: db) - - row = { - **_AVITO_FULL_LOAD_ROW, - "default_params": {**_AVITO_FULL_LOAD_ROW["default_params"], "incremental_days": 2}, - } - await _sched.trigger_avito_full_load_run(db, row) - await asyncio.sleep(0) - - assert len(calls["load"]) == 1 - assert calls["load"][0]["incremental_days"] == 2 - - -@pytest.mark.asyncio -async def test_trigger_avito_full_load_no_incremental_days_is_none( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Без incremental_days в params → run_avito_full_load(incremental_days=None) (exhaustive).""" - db = _FakeSchedulerDB() - calls: dict[str, list] = {"load": []} - - def fake_create_run(_d, *, source, params): - db.running[source] = True - return 411 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - async def fake_full_load(run_db, *, run_id, **kwargs): - calls["load"].append(kwargs) - - monkeypatch.setattr(_sched, "run_avito_full_load", fake_full_load) - monkeypatch.setattr(_sched, "SessionLocal", lambda: db) - - await _sched.trigger_avito_full_load_run(db, _AVITO_FULL_LOAD_ROW) - await asyncio.sleep(0) - - assert len(calls["load"]) == 1 - assert calls["load"][0]["incremental_days"] is None - - -_AVITO_FULL_LOAD_EXHAUSTIVE_ROW = { - "source": "avito_full_load_exhaustive", - "window_start_hour": 13, - "window_end_hour": 15, - "default_params": { - "price_cap_per_bucket": 1400, - "concurrency": 1, - "request_delay_sec": 7.0, - "secondary_only": True, - }, -} - - -@pytest.mark.asyncio -async def test_trigger_avito_full_load_exhaustive_forces_none( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """avito_full_load_exhaustive форсит incremental_days=None даже если в params задан.""" - db = _FakeSchedulerDB() - calls: dict[str, list] = {"load": []} - - def fake_create_run(_d, *, source, params): - db.running[source] = True - return 420 - - monkeypatch.setattr(_sched.runs_mod, "create_run", fake_create_run) - - async def fake_full_load(run_db, *, run_id, **kwargs): - calls["load"].append((run_id, kwargs)) - - monkeypatch.setattr(_sched, "run_avito_full_load", fake_full_load) - monkeypatch.setattr(_sched, "SessionLocal", lambda: db) - - # Даже если оператор случайно положит incremental_days в params — exhaustive игнорит. - row = { - **_AVITO_FULL_LOAD_EXHAUSTIVE_ROW, - "default_params": { - **_AVITO_FULL_LOAD_EXHAUSTIVE_ROW["default_params"], - "incremental_days": 5, - }, - } - run_id = await _sched.trigger_avito_full_load_exhaustive_run(db, row) - await asyncio.sleep(0) - - assert run_id == 420 - assert len(calls["load"]) == 1 - loaded_run_id, kwargs = calls["load"][0] - assert loaded_run_id == 420 - assert kwargs["incremental_days"] is None - assert kwargs["concurrency"] == 1 - assert kwargs["price_cap_per_bucket"] == 1400 - assert kwargs["resume_run_id"] is None - - -@pytest.mark.asyncio -async def test_scheduler_dispatch_routes_avito_full_load_exhaustive( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Dispatch роутит avito_full_load_exhaustive → trigger_avito_full_load_exhaustive_run.""" - triggered: list[str] = [] - - async def fake_trigger(db, sch): - triggered.append(sch["source"]) - return 1 - - monkeypatch.setattr(_sched, "trigger_avito_full_load_exhaustive_run", fake_trigger) - - db = object() - sch = _AVITO_FULL_LOAD_EXHAUSTIVE_ROW - source = sch["source"] - # Replicate the elif chain from scheduler_loop - if source == "avito_full_load": - pass - elif source == "avito_full_load_exhaustive": - await _sched.trigger_avito_full_load_exhaustive_run(db, sch) - - assert "avito_full_load_exhaustive" in triggered - - -# --------------------------------------------------------------------------- -# #1182 Phase 2 — graceful SIGTERM-drain детей (detached run-задач) -# --------------------------------------------------------------------------- - - -@pytest.mark.asyncio -async def test_drain_inflight_awaits_detached_cooperative_child() -> None: - """Гвоздь Phase 2: _drain_inflight ДОЖИДАЕТСЯ detached-ребёнка (как scrape-run, - отвязанный через _spawn_tracked) до его clean-exit, а НЕ хард-кансельит его. - - Регрессия gap'а: coordinator раньше ждал только себя → asyncio.run teardown - хард-кансельил живых детей mid-await (#1182 failure mode). - """ - clean_exit = asyncio.Event() # == mark_done-эквивалент ребёнка - - async def _child() -> None: - # Кооперативный ребёнок: крутится, пока не запрошен shutdown, потом выходит чисто. - while not sd.shutdown_requested(): - await asyncio.sleep(0.01) - clean_exit.set() - - child = sched._spawn_tracked(_child()) # detached, попадает в _inflight_tasks - await asyncio.sleep(0.02) - assert not child.done(), "ребёнок ещё бежит (shutdown не запрошен)" - - sd.request_shutdown() - await asyncio.wait_for(sched._drain_inflight(), timeout=2.0) - - assert clean_exit.is_set(), "drain дождался clean-exit ребёнка" - assert child.done() - assert not child.cancelled(), "ребёнок вышел сам, НЕ хард-cancel" - - -@pytest.mark.asyncio -async def test_drain_inflight_times_out_on_noncooperative_child( - monkeypatch: pytest.MonkeyPatch, -) -> None: - """Некооперативный ребёнок (игнорит shutdown) упирается в _CHILD_DRAIN_TIMEOUT_S и - остаётся бежать — отдаётся внешнему hard-cancel fallback (scheduler_main/docker).""" - monkeypatch.setattr(sched, "_CHILD_DRAIN_TIMEOUT_S", 0.05) - - async def _stuck() -> None: - await asyncio.Event().wait() # никогда не проверяет shutdown_requested() - - child = sched._spawn_tracked(_stuck()) - sd.request_shutdown() - - await asyncio.wait_for(sched._drain_inflight(), timeout=2.0) - - assert not child.done(), "drain НЕ повесил процесс — оставил ребёнка под hard-cancel" - - # cleanup - child.cancel() - with pytest.raises(asyncio.CancelledError): - await child - - -@pytest.mark.asyncio -async def test_drain_inflight_noop_when_no_children() -> None: - """Без in-flight детей _drain_inflight() — мгновенный no-op (без сна/ожидания).""" - assert not sched._inflight_tasks - await asyncio.wait_for(sched._drain_inflight(), timeout=1.0) - - -@pytest.mark.asyncio -async def test_scheduler_loop_calls_drain_on_shutdown(monkeypatch: pytest.MonkeyPatch) -> None: - """scheduler_loop при выходе по SIGTERM-drain'у вызывает _drain_inflight (проводка - coordinator → дренаж детей). Initial sleep(30) и tick-sleep занулены, чтобы тест - был детерминированным и быстрым.""" - drain_spy = AsyncMock() - monkeypatch.setattr(sched, "_drain_inflight", drain_spy) - monkeypatch.setattr(sched.asyncio, "sleep", AsyncMock()) # без реальных 30s/60s - - sd.request_shutdown() # первый же top-of-tick check → break - - await asyncio.wait_for(sched.scheduler_loop(), timeout=2.0) - - drain_spy.assert_awaited_once() diff --git a/tradein-mvp/backend/tests/test_scheduler_main.py b/tradein-mvp/backend/tests/test_scheduler_main.py index a8c49700..d74c378a 100644 --- a/tradein-mvp/backend/tests/test_scheduler_main.py +++ b/tradein-mvp/backend/tests/test_scheduler_main.py @@ -56,15 +56,13 @@ async def test_run_cancels_cleanly(monkeypatch: pytest.MonkeyPatch) -> None: cancelled = asyncio.Event() async def _stub_loop() -> None: - """Короткая заглушка: ждёт отмены, имитируя бесконечный scheduler_loop.""" + """Короткая заглушка: ждёт отмены, имитируя бесконечный kit scheduler_loop.""" cancelled.set() await asyncio.sleep(999) - # Патчим источник импорта: _run() делает `from app.services.scheduler import scheduler_loop` - # на каждый вызов, поэтому патч модуля-источника достаточен. - import app.services.scheduler as _sched_mod - - monkeypatch.setattr(_sched_mod, "scheduler_loop", _stub_loop) + # #2397 Part C: _run() безусловно зовёт _run_kit_scheduler() (legacy fallback удалён) — + # патчим саму функцию на модуле scheduler_main. + monkeypatch.setattr(sm, "_run_kit_scheduler", _stub_loop) task = asyncio.create_task(sm._run()) @@ -139,9 +137,8 @@ async def test_run_uses_request_shutdown_not_cancel(monkeypatch: pytest.MonkeyPa while not sd.shutdown_requested(): await asyncio.sleep(0.01) - import app.services.scheduler as _sched_mod - - monkeypatch.setattr(_sched_mod, "scheduler_loop", _coop_loop) + # #2397 Part C: _run() безусловно зовёт _run_kit_scheduler() (legacy fallback удалён). + monkeypatch.setattr(sm, "_run_kit_scheduler", _coop_loop) run_task = asyncio.create_task(sm._run()) await asyncio.wait_for(started.wait(), timeout=2.0) diff --git a/tradein-mvp/backend/tests/test_scraper_kit_scheduler_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_scheduler_parity.py index 29b7e84c..38429830 100644 --- a/tradein-mvp/backend/tests/test_scraper_kit_scheduler_parity.py +++ b/tradein-mvp/backend/tests/test_scraper_kit_scheduler_parity.py @@ -1,16 +1,13 @@ -"""Golden-parity: `scraper_kit.orchestration.scheduler` registry-dispatch ≡ старый if/elif. +"""Smoke/shape tests for `scraper_kit.orchestration.scheduler` (#2136, post-strangler). -Strangler-инвариант (#2136): registry-реестр (`build_registry` + `resolve_handler` + -`_dispatch`) должен маршрутизировать те же source'ы к тем же обработчикам, что и боевой -27-веточный `if/elif` в `app.services.scheduler.scheduler_loop`, и сохранить критичную -concurrency-логику (`_claim_run` advisory-lock + double-check, `reap_zombies` порог, -proxy_healthcheck sub-hourly post-claim). +Kit — единственный scheduler-путь (#2397 Part C убрал legacy +`app.services.scheduler.scheduler_loop` + trigger_*/get_due_schedules/_claim_run/ +reap_zombies machinery; сравнивать «golden-parity» больше не с чем). Этот файл проверяет +форму и инварианты kit-реестра самого по себе: -Метод: - 1. ROUTING: для каждого source прогоняем ОДИН тик боевого `scheduler_loop` (все - `trigger_*` замоканы) → фиксируем какой trigger сработал; параллельно `resolve_handler` - kit-реестра обязан вернуть handler для того же source. Множества покрытых source'ов - обязаны совпасть → таблица маршрутизации эквивалентна. + 1. REGISTRY: реальный `build_product_handlers()` (НЕ stub) + `build_registry` покрывает + каждый canonical scheduled source (kit-native + продуктовые + deactivate_stale_* + wildcard-члены); неизвестный source → None. 2. CLAIM: `_claim_run` — happy-path, already-running skip, lock-busy skip, appeared-under- lock rollback. Мокаем БД + ctx.runs. 3. ZOMBIE: `reap_zombies` возвращает число + commit. @@ -25,12 +22,9 @@ from __future__ import annotations import asyncio import os -from contextlib import ExitStack from typing import Any from unittest.mock import AsyncMock, MagicMock, patch -import pytest - os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from scraper_kit.orchestration import scheduler as kit_sched @@ -45,45 +39,41 @@ from scraper_kit.orchestration.scheduler import ( resolve_handler, ) -import app.services.scheduler as old_sched - -# ── таблица маршрутизации: source → имя боевого trigger_* ───────────────────── -# (то, что `if/elif` в старом scheduler_loop выбирает для каждого source). -SOURCE_TO_OLD_TRIGGER: dict[str, str] = { - "avito_city_sweep": "trigger_avito_city_sweep_run", - "avito_full_load": "trigger_avito_full_load_run", - "avito_full_load_exhaustive": "trigger_avito_full_load_exhaustive_run", - "avito_newbuilding_sweep": "trigger_avito_newbuilding_sweep_run", - "yandex_city_sweep": "trigger_yandex_city_sweep_run", - "cian_history_backfill": "trigger_cian_backfill_run", - "cian_city_sweep": "trigger_cian_city_sweep_run", - "cian_full_load": "trigger_cian_full_load_run", - "domclick_city_sweep": "trigger_domclick_city_sweep_run", - "rosreestr_dkp_import": "trigger_rosreestr_dkp_run", - "listing_source_snapshot": "trigger_listing_source_snapshot_run", - "asking_to_sold_ratio_refresh": "trigger_asking_to_sold_ratio_run", - "refresh_search_matview": "trigger_refresh_search_matview_run", - "yandex_address_backfill": "trigger_yandex_address_backfill_run", - "deactivate_stale_avito": "trigger_deactivate_stale_run", - "deactivate_stale_yandex": "trigger_deactivate_stale_run", - "deactivate_stale_cian": "trigger_deactivate_stale_run", - "sber_index_pull": "trigger_sber_index_pull_run", - "rosreestr_quarter_poll": "trigger_rosreestr_quarter_poll_run", - "newbuilding_enrich": "trigger_newbuilding_enrich_run", - "yandex_newbuilding_sweep": "trigger_yandex_newbuilding_sweep_run", - "geoportal_coords_backfill": "trigger_geoportal_coords_backfill_run", - "geocode_missing_listings": "trigger_geocode_missing_listings_run", - "avito_detail_backfill": "trigger_avito_detail_backfill_run", - "yandex_detail_backfill": "trigger_yandex_detail_backfill_run", - "cadastral_geo_match": "trigger_cadastral_geo_match_run", - "house_imv_backfill": "trigger_house_imv_backfill_run", - "house_dedup_merge": "trigger_house_dedup_merge_run", - "proxy_healthcheck": "trigger_proxy_healthcheck_run", +# ── продуктовые source'ы (НЕ kit-native) — CANONICAL, поддерживается ВРУЧНУЮ ───────── +# Источник истины: app.services.product_handlers.build_product_handlers() keys (минус +# wildcard "deactivate_stale_*", раскрытый в 3 конкретных member-source'а) + семейство +# scrape_schedules-сидов (data/sql/*scrape_schedules*seed*.sql, 158_seed_proxy_healthcheck_ +# schedule.sql, 162_seed_deals_freshness_monitor.sql). Этот список НЕ выводится из кода — +# добавил/убрал источник в product_handlers.py? Обнови и здесь, иначе +# test_real_build_product_handlers_covers_all_scheduled_sources ничего не поймает. +# (2026-07-04 deep-review #2397 Part C: список был устаревшим — не хватало +# cian_history_backfill/deals_freshness_monitor/osm_poi_ekb_refresh; исправлено.) +_PRODUCT_SOURCES: set[str] = { + "cian_history_backfill", + "rosreestr_dkp_import", + "listing_source_snapshot", + "asking_to_sold_ratio_refresh", + "refresh_search_matview", + "yandex_address_backfill", + "deactivate_stale_avito", + "deactivate_stale_yandex", + "deactivate_stale_cian", + "sber_index_pull", + "rosreestr_quarter_poll", + "deals_freshness_monitor", + "newbuilding_enrich", + "yandex_newbuilding_sweep", + "geoportal_coords_backfill", + "geocode_missing_listings", + "avito_detail_backfill", + "yandex_detail_backfill", + "cadastral_geo_match", + "osm_poi_ekb_refresh", + "house_imv_backfill", + "house_dedup_merge", + "proxy_healthcheck", } -# все distinct trigger-имена, которые надо замокать в боевом scheduler_loop -_ALL_TRIGGER_NAMES = sorted(set(SOURCE_TO_OLD_TRIGGER.values())) - # kit-native (тело в scraper_kit.orchestration.pipeline) — регистрируются встроенно _KIT_NATIVE_SOURCES = { "avito_city_sweep", @@ -129,7 +119,7 @@ def _build_recording_registry() -> tuple[dict[str, Handler], dict[str, MagicMock product: dict[str, Handler] = {} # продуктовые exact-match источники - for src in SOURCE_TO_OLD_TRIGGER: + for src in _PRODUCT_SOURCES: if src in _KIT_NATIVE_SOURCES or src.startswith("deactivate_stale_"): continue product[src] = Handler(_make_job(src), src) @@ -139,56 +129,30 @@ def _build_recording_registry() -> tuple[dict[str, Handler], dict[str, MagicMock return build_registry(product), fired -# ── 1. ROUTING parity ──────────────────────────────────────────────────────── +# ── 1. REGISTRY shape ──────────────────────────────────────────────────────── -async def _drive_old_one_tick(source: str) -> list[str]: - """Прогнать ОДИН тик боевого scheduler_loop для одного source, вернуть сработавшие trigger'ы.""" - sch = _make_sched(source) - triggers = {name: AsyncMock(return_value=1) for name in _ALL_TRIGGER_NAMES} +def test_real_build_product_handlers_covers_all_scheduled_sources() -> None: + """Реальный (не stub!) build_product_handlers покрывает КАЖДЫЙ canonical source. - with ExitStack() as es: - es.enter_context(patch.object(old_sched, "reap_zombies", MagicMock())) - es.enter_context( - patch.object(old_sched, "get_due_schedules", MagicMock(return_value=[sch])) - ) - es.enter_context(patch.object(old_sched, "SessionLocal", MagicMock())) - es.enter_context(patch.object(old_sched.asyncio, "sleep", AsyncMock())) - # shutdown: False на входе while, затем True после первого dispatch → выход - es.enter_context( - patch.object( - old_sched, "shutdown_requested", MagicMock(side_effect=[False] + [True] * 10) - ) - ) - for name, mock in triggers.items(): - es.enter_context(patch.object(old_sched, name, mock)) - await old_sched.scheduler_loop() + 2026-07-04 deep-review #2397 Part C: старая версия этого теста строила registry ИЗ + `_PRODUCT_SOURCES` через recording-stub (`_build_recording_registry`) и проверяла, что + тот же набор резолвится — тавтология, ничего не гоняющая через реальный код. Эта версия + зовёт настоящий `product_handlers.build_product_handlers(ctx)` — если кто-то уронит + handler-entry в product_handlers.py (или source добавили в scrape_schedules seed, но + забыли завести handler), тест падает. - return [name for name, m in triggers.items() if m.await_count > 0] + ctx=None безопасен: build_product_handlers его не замыкает при сборке dict (сами job'ы + получают ctx только во время dispatch) — см. её докстринг + test_deals_freshness_monitor. + """ + from app.services.product_handlers import build_product_handlers + real_registry = build_registry(build_product_handlers(ctx=None)) # type: ignore[arg-type] -@pytest.mark.parametrize("source", sorted(SOURCE_TO_OLD_TRIGGER)) -async def test_routing_parity_per_source(source: str) -> None: - """Для каждого source: старый if/elif зовёт ожидаемый trigger, kit-реестр резолвит handler.""" - expected_trigger = SOURCE_TO_OLD_TRIGGER[source] - - # старый роутинг - fired = await _drive_old_one_tick(source) - assert fired == [ - expected_trigger - ], f"source={source}: old scheduler fired {fired}, expected [{expected_trigger}]" - - # kit роутинг — тот же source обязан резолвиться в handler - registry, _ = _build_recording_registry() - handler = resolve_handler(source, registry) - assert handler is not None, f"kit registry has no handler for source={source}" - - -def test_routing_coverage_sets_match() -> None: - """Множество source'ов kit-реестра ⊇ множество, покрытое боевым if/elif.""" - registry, _ = _build_recording_registry() - for source in SOURCE_TO_OLD_TRIGGER: - assert resolve_handler(source, registry) is not None, f"kit misses source={source}" + for source in _PRODUCT_SOURCES | _KIT_NATIVE_SOURCES: + assert ( + resolve_handler(source, real_registry) is not None + ), f"real build_product_handlers()/build_registry() misses source={source}" def test_kit_native_handler_set() -> None: @@ -342,7 +306,7 @@ def test_reap_zombies_counts_and_commits() -> None: def test_reap_zombies_threshold_constant() -> None: - assert kit_sched.ZOMBIE_THRESHOLD_HOURS == old_sched.ZOMBIE_THRESHOLD_HOURS == 6 + assert kit_sched.ZOMBIE_THRESHOLD_HOURS == 6 # ── 4. proxy_healthcheck sub-hourly post_claim ───────────────────────────────