gendesign/tradein-mvp/backend/app/services/scheduler.py

"""In-app scheduler — periodically triggers configured scrapes based on scrape_schedules table.

Запускается в FastAPI lifespan (app.main lifespan context manager) при startup.
Логика:
1. Каждые 60 sec query scrape_schedules WHERE enabled=true AND next_run_at <= NOW().
2. Для каждого: проверить нет ли уже running run этого source — если нет, trigger.
3. После trigger — recompute next_run_at в следующих суток.
4. На old runs (orphaned status='running' > 6h без heartbeat) — пометить как 'zombie'.

Sources:
  - avito_city_sweep        → run_avito_city_sweep (scrape_pipeline.py)
  - yandex_city_sweep       → run_yandex_city_sweep (scrape_pipeline.py, #561; shipped DORMANT)
  - n1_city_sweep           → run_n1_city_sweep (scrape_pipeline.py, #575)
  - cian_history_backfill   → run_cian_backfill (this module, #560)
  - rosreestr_dkp_import    → import_rosreestr_dkp (this module, #563)
  - listing_source_snapshot → snapshot_listing_sources (tasks/listing_source_snapshot.py, #570;
                              pure internal DB snapshot — no external calls, enabled by default)
  - asking_to_sold_ratio_refresh → recompute_asking_to_sold_ratios
                              (tasks/asking_to_sold_ratio.py, #648 Stage 4; pure internal DB
                              re-derivation of the asking→sold ratios — no external calls,
                              enabled by default; window after rosreestr_dkp_import)
  - yandex_address_backfill  → run_yandex_address_backfill
                              (tasks/yandex_address_backfill.py, #855; EKB-pilot: fetches
                              yandex detail pages, extracts full address from <title>,
                              updates listings.address for street-only addresses;
                              nightly 02:00-03:00 UTC, before refresh_search_matview)
  - deactivate_stale_avito  → deactivate_stale_avito_listings
                              (tasks/deactivate_stale_avito.py, #759; marks avito listings
                              with last_seen_at > TTL days as is_active=false — no DELETE,
                              no external calls, enabled by default; window after avito sweep)
  - sber_index_pull         → run_sber_index_pull
                              (tasks/sber_index_pull.py, #887; monthly pull of СберИндекс
                              city-level secondary-housing price index — pure HTTP+DB, no
                              anti-bot/proxy; window 05:00-06:00 UTC, ~28-day cadence)
  - rosreestr_quarter_poll  → run_rosreestr_quarter_poll
                              (tasks/rosreestr_quarter_poll.py, #888; monthly HEAD-check of
                              Rosreestr open-data portal for new quarter availability — pure
                              HTTP detect+alert, NO download/shell-out; window 07:00-08:00 UTC,
                              ~28-day cadence)
  - newbuilding_enrich      → run_newbuilding_enrich
                              (tasks/newbuilding_enrich_backfill.py, #973; nightly slice of the
                              cian_newbuilding houses enrichment — resolves cian_zhk_url, fetches
                              the ЖК page, persists houses_price_dynamics / house_reliability_checks
                              / house_reviews. Cian HTTP (curl_cffi chrome120) + anti-bot delay.
                              Idempotent: skips houses already enriched, per-house SAVEPOINT, so
                              each fire drains the next `limit` pending houses; window 00:00-01:00
                              UTC = 03:00-04:00 МСК, before the 01:00+ UTC sweep block)
  - yandex_newbuilding_sweep → enrich_yandex_newbuilding_sweep
                              (tasks/yandex_newbuilding_sweep.py, #974; enrichment ЖК в
                              market.yandex_jk_enrichment через BrowserFetcher; shipped DORMANT —
                              enable manually after deploy verified; window 02:00-05:00 UTC)
  - cian_city_sweep         → run_cian_city_sweep (newbuilding Phase 4, #973; shipped DORMANT —
                              proxy dead, enable manually after proxy restored)
  - geocode_missing_listings → run_geocode_missing_listings
                              (tasks/geocode_missing.py; nightly backfill listings.lat/lon для
                              всех источников с NULL coords; address-dedup batch loop с
                              wall-clock budget; window 06:00-09:00 UTC — после city sweeps
                              ~03-04 UTC и deals OS-cron 05:30/05:45 UTC)
  - yandex_detail_backfill  → run_yandex_detail_backfill
                              (tasks/yandex_detail_backfill.py, #1553; nightly backfill of
                              yandex detail_enriched_at for ~3952 listings never enriched by
                              city_sweep; fetch via curl_cffi chrome120 + scraper_proxy_url,
                              parse via YandexDetailScraper.parse;
                              window 12:00-15:00 UTC — после avito_detail_backfill 09-12 UTC)
"""

from __future__ import annotations

import asyncio
import logging
import random
from datetime import UTC, datetime, time, timedelta
from typing import Any

import sentry_sdk
from sqlalchemy import text
from sqlalchemy.orm import Session

from app.core.db import SessionLocal
from app.services import scrape_runs as runs_mod
from app.services.scrape_pipeline import (
    run_avito_city_sweep,
    run_cian_city_sweep,
    run_n1_city_sweep,
    run_yandex_city_sweep,
)

logger = logging.getLogger(__name__)

# Loop interval — check каждую минуту
SCHEDULER_TICK_SEC = 60
ZOMBIE_THRESHOLD_HOURS = 6


def compute_next_run_at(
    window_start_hour: int, window_end_hour: int, *, now: datetime | None = None
) -> datetime:
    """Pick random datetime в window [start, end) UTC, для СЛЕДУЮЩИХ суток после now.

    Если window_end_hour <= window_start_hour → cross-midnight window
    (например 22→3 → окно 22:00-23:59 ИЛИ 00:00-02:59).
    """
    now = now or datetime.now(tz=UTC)
    tomorrow = (now + timedelta(days=1)).date()

    if window_end_hour > window_start_hour:
        # Обычное окно (например 2..5 → 02:00-04:59)
        start_seconds = window_start_hour * 3600
        end_seconds = window_end_hour * 3600
        rand_seconds = random.randint(start_seconds, end_seconds - 1)
        return datetime.combine(tomorrow, time(0, 0), tzinfo=UTC) + timedelta(seconds=rand_seconds)
    else:
        # Cross-midnight (22..3 → 22:00-23:59 + 00:00-02:59)
        # Длина окна = (24-start) + end часов
        total_seconds = ((24 - window_start_hour) + window_end_hour) * 3600
        rand_seconds = random.randint(0, total_seconds - 1)
        # Если rand попадает в первую часть (start..24)
        first_half = (24 - window_start_hour) * 3600
        if rand_seconds < first_half:
            # Текущая дата (если ещё не наступило окно) или next day
            base_date = now.date() if now.hour < window_start_hour else tomorrow
            return datetime.combine(base_date, time(0, 0), tzinfo=UTC) + timedelta(
                seconds=window_start_hour * 3600 + rand_seconds
            )
        else:
            # Во второй части (0..end), следующего дня
            offset = rand_seconds - first_half
            return datetime.combine(tomorrow, time(0, 0), tzinfo=UTC) + timedelta(seconds=offset)


def has_running_run(db: Session, source: str) -> bool:
    """Есть ли активный run для source (status='running')."""
    row = db.execute(
        text(
            """
            SELECT 1 FROM scrape_runs
            WHERE source = :source AND status = 'running'
            LIMIT 1
            """
        ),
        {"source": source},
    ).fetchone()
    return row is not None


def reap_zombies(db: Session) -> int:
    """Mark scrape_runs as 'zombie' если heartbeat не обновлялся > ZOMBIE_THRESHOLD_HOURS hours."""
    zombie_interval = f"{ZOMBIE_THRESHOLD_HOURS} hours"
    result = db.execute(
        text(
            """
            UPDATE scrape_runs
            SET status = 'zombie', finished_at = NOW()
            WHERE status = 'running'
              AND (heartbeat_at IS NULL
                   OR heartbeat_at < NOW() - CAST(:interval AS interval))
            RETURNING id
            """
        ),
        {"interval": zombie_interval},
    )
    rows = result.fetchall()
    db.commit()
    if rows:
        logger.warning("scheduler: reaped %d zombie runs: %s", len(rows), [r.id for r in rows])
    return len(rows)


def _claim_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
    """Claim run: INSERT scrape_runs + UPDATE scrape_schedules next_run_at.

    Returns run_id или None если уже есть running run для этого source.
    Общий helper для всех source-handler'ов.
    """
    source = schedule_row["source"]
    if has_running_run(db, source):
        logger.info("scheduler: skip — already running for source=%s", source)
        return None

    # #750: атомарный claim через transaction-scoped advisory lock. Сериализует
    # конкурентные _claim_run одного source (redeploy-overlap старого+нового контейнера
    # / 2 реплики), иначе оба пройдут has_running_run-pre-check и запустят ДВОЙНОЙ sweep
    # → 2× rate к Avito/Cian → бан. pg_try_advisory_xact_lock НЕ блокирует: lock занят
    # другим тиком → false → return None. Лок авто-снимается на commit/rollback этой
    # транзакции (has_running_run выше — быстрый pre-check, обычно без взятия лока).
    got_lock = db.execute(
        text("SELECT pg_try_advisory_xact_lock(hashtext(:source))"),
        {"source": source},
    ).scalar()
    if not got_lock:
        logger.info("scheduler: skip — concurrent claim in progress for source=%s", source)
        return None

    # Double-checked под локом: конкурентный тик мог закоммитить running-run МЕЖДУ
    # pre-check и взятием лока (затем освободить lock на commit). Перепроверяем под
    # локом — иначе словим двойной run на узком окне.
    if has_running_run(db, source):
        logger.info("scheduler: skip — running appeared under lock for source=%s", source)
        db.rollback()  # освобождаем advisory lock (claim не состоялся)
        return None

    params = schedule_row.get("default_params") or {}
    run_id = runs_mod.create_run(db, source=source, params=params)

    next_at = compute_next_run_at(
        schedule_row["window_start_hour"],
        schedule_row["window_end_hour"],
    )
    db.execute(
        text(
            """
            UPDATE scrape_schedules
            SET last_run_id = :run_id, last_run_at = NOW(),
                next_run_at = :next_at, updated_at = NOW()
            WHERE source = :source
            """
        ),
        {"run_id": run_id, "next_at": next_at, "source": source},
    )
    db.commit()
    logger.info(
        "scheduler: claimed run_id=%d source=%s next_run_at=%s",
        run_id,
        source,
        next_at.isoformat(),
    )
    return run_id


def _defer_next_run_at(db: Session, schedule_row: dict[str, Any]) -> None:
    """Сдвинуть next_run_at на следующее окно БЕЗ создания run (#1522).

    Используется когда trigger делает early-return (например cian: отсутствуют/протухли
    cookies) ДО _claim_run. Без этого get_due_schedules переотбирает schedule на каждом
    тике (SCHEDULER_TICK_SEC), и pre-check (verify_session → реальный HTTP к Cian) гоняется
    раз в минуту круглосуточно. Сдвиг откладывает следующую попытку до завтрашнего окна.
    """
    source = schedule_row["source"]
    next_at = compute_next_run_at(
        schedule_row["window_start_hour"],
        schedule_row["window_end_hour"],
    )
    db.execute(
        text(
            """
            UPDATE scrape_schedules
            SET next_run_at = :next_at, updated_at = NOW()
            WHERE source = :source
            """
        ),
        {"next_at": next_at, "source": source},
    )
    db.commit()
    logger.info(
        "scheduler: deferred next_run_at source=%s next_run_at=%s (no run claimed)",
        source,
        next_at.isoformat(),
    )


async def trigger_avito_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
    """Создать новый scrape_runs + launch run_avito_city_sweep в asyncio.create_task.

    Returns run_id (или None если skip — есть running run).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    # Spawn asyncio task — pass NEW session (avoid sharing with this scheduler tick)
    async def _run() -> None:
        run_db = SessionLocal()
        try:
            await run_avito_city_sweep(
                run_db,
                run_id=run_id,
                pages_per_anchor=int(params.get("pages_per_anchor", 3)),
                detail_top_n=int(params.get("detail_top_n", 20)),
                request_delay_sec=float(params.get("request_delay_sec", 7.0)),
                enrich_houses=bool(params.get("enrich_houses", True)),
                radius_m=int(params.get("radius_m", 1500)),
            )
        except Exception:
            logger.exception("scheduler: run_avito_city_sweep crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    # Keep reference to avoid GC before task completes (RUF006)
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered city_sweep run_id=%d", run_id)
    return run_id


async def trigger_yandex_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
    """Создать новый scrape_runs + launch run_yandex_city_sweep в asyncio.create_task.

    Зеркало trigger_avito_city_sweep_run, но проще (#561): у Yandex sweep нет
    houses/detail-параметров. Shipped DORMANT — schedule seed enabled=false (078),
    включается оператором вручную после #623 (egress-IP rotation).

    Returns run_id (или None если skip — есть running run).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    # Spawn asyncio task — pass NEW session (avoid sharing with this scheduler tick)
    async def _run() -> None:
        run_db = SessionLocal()
        try:
            await run_yandex_city_sweep(
                run_db,
                run_id=run_id,
                pages_per_anchor=int(params.get("pages_per_anchor", 2)),
                request_delay_sec=float(params.get("request_delay_sec", 9.0)),
                radius_m=int(params.get("radius_m", 1500)),
                enrich_address=bool(params.get("enrich_address", True)),
            )
        except Exception:
            logger.exception("scheduler: run_yandex_city_sweep crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    # Keep reference to avoid GC before task completes (RUF006)
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered yandex_city_sweep run_id=%d", run_id)
    return run_id


async def trigger_n1_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
    """Создать новый scrape_runs + launch run_n1_city_sweep в asyncio.create_task (#575).

    Зеркало trigger_yandex_city_sweep_run — N1 sweep простой (search → save, без
    houses/detail). Returns run_id (или None если skip — есть running run).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            await run_n1_city_sweep(
                run_db,
                run_id=run_id,
                pages_per_anchor=int(params.get("pages_per_anchor", 2)),
                request_delay_sec=float(params.get("request_delay_sec", 7.0)),
                radius_m=int(params.get("radius_m", 1500)),
            )
        except Exception:
            logger.exception("scheduler: run_n1_city_sweep crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered n1_city_sweep run_id=%d", run_id)
    return run_id


async def trigger_cian_backfill_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
    """Создать scrape_runs + launch run_cian_backfill в asyncio.create_task.

    Pre-checks Cian session validity before claiming the run.
    Marks nothing as failed if cookies are absent/expired — just logs + GlitchTip warning.
    Returns run_id или None (skip / expired cookies).
    """
    from app.services.cian_session import load_session, verify_session

    # Pre-check: load and verify cookies before claiming the run slot
    cookies = load_session(db)
    if cookies is None:
        logger.warning("scheduler: cian_history_backfill skipped — no valid session cookies in DB")
        # #1522: всё равно сдвигаем next_run_at на следующее окно, иначе get_due_schedules
        # переотбирает этот schedule на КАЖДОМ тике (60с) и verify_session долбит Cian
        # круглосуточно, пока оператор вручную не зальёт свежие cookies — burn proxy-квоты
        # + риск TLS/bot-бана.
        _defer_next_run_at(db, schedule_row)
        return None

    state = await verify_session(cookies)
    if state is None:
        logger.warning(
            "scheduler: cian_history_backfill — cookies expired or invalid, skipping run"
        )
        try:
            sentry_sdk.capture_message(
                "cian_history_backfill skipped: Cian session cookies expired — "
                "please re-upload via admin UI",
                level="warning",
            )
        except Exception:
            pass  # sentry_sdk not initialised in dev
        # #1522: cookies протухли — сдвигаем next_run_at (см. комментарий выше), чтобы
        # verify_session не бил по Cian каждые 60с до ручной перезаливки cookies.
        _defer_next_run_at(db, schedule_row)
        return None

    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            await _execute_cian_backfill(run_db, run_id=run_id, params=params)
        except Exception:
            logger.exception("scheduler: _execute_cian_backfill crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered cian_history_backfill run_id=%d", run_id)
    return run_id


async def trigger_cian_city_sweep_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
    """Создать scrape_runs + launch run_cian_city_sweep в asyncio.create_task (#973).

    Несёт Phase 4 newbuilding-enrichment (enrich_houses). Shipped DORMANT — seed
    enabled=false (107), включается оператором вручную после восстановления прокси.
    Returns run_id (или None если skip — есть running run).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None
    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            await run_cian_city_sweep(
                run_db,
                run_id=run_id,
                pages_per_anchor=int(params.get("pages_per_anchor", 3)),
                request_delay_sec=float(params.get("request_delay_sec", 5.0)),
                radius_m=int(params.get("radius_m", 1500)),
                detail_top_n=int(params.get("detail_top_n", 10)),
                enrich_houses=bool(params.get("enrich_houses", True)),
            )
        except Exception:
            logger.exception("scheduler: run_cian_city_sweep crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered cian_city_sweep run_id=%d", run_id)
    return run_id


async def _execute_cian_backfill(
    db: Session,
    *,
    run_id: int,
    params: dict[str, Any],
) -> None:
    """Orchestrate Cian history backfill with heartbeat + checkpoint.

    Wraps backfill_cian_history(), updating scrape_runs counters (via update_heartbeat)
    before and after the batch call for zombie-detection visibility.

    Checkpoint/resume semantics: backfill_cian_history() queries rows WHERE history IS
    NULL via LEFT JOIN — so re-running after a partial completion naturally skips
    already-processed rows (idempotent by design).

    Params (from default_params jsonb):
        batch_size: int — rows per run (listings + houses counted separately).
    """
    from app.tasks.cian_history_backfill import backfill_cian_history

    batch_size = int(params.get("batch_size", 100))
    counters: dict[str, int] = {
        "listings_processed": 0,
        "listings_succeeded": 0,
        "listings_failed": 0,
        "houses_processed": 0,
        "houses_succeeded": 0,
        "houses_failed": 0,
    }

    try:
        runs_mod.update_heartbeat(db, run_id, counters)

        result = await backfill_cian_history(
            db,
            batch_size=batch_size,
            do_listings=True,
            do_houses=True,
            do_valuations=False,
        )

        counters = {
            "listings_processed": result.listings_processed,
            "listings_succeeded": result.listings_succeeded,
            "listings_failed": result.listings_failed_fetch + result.listings_failed_save,
            "houses_processed": result.houses_processed,
            "houses_succeeded": result.houses_succeeded,
            "houses_failed": result.houses_failed_fetch + result.houses_failed_save,
            "duration_sec": int(result.duration_sec),
        }
        runs_mod.mark_done(db, run_id, counters)
        logger.info(
            "scheduler: cian_history_backfill run_id=%d done — listings=%d/%d houses=%d/%d %.1fs",
            run_id,
            result.listings_succeeded,
            result.listings_total,
            result.houses_succeeded,
            result.houses_total,
            result.duration_sec,
        )
    except Exception as exc:
        logger.exception("scheduler: cian_history_backfill run_id=%d failed", run_id)
        runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters)
        raise


async def trigger_yandex_address_backfill_run(
    db: Session, schedule_row: dict[str, Any]
) -> int | None:
    """Создать scrape_runs + launch run_yandex_address_backfill в asyncio.create_task.

    EKB-only pilot (#855): backfills listings.address for yandex listings that have
    street-only addresses (no house number). Fetches detail pages via curl_cffi chrome120,
    extracts full address from <title>, updates listings WHERE address differs.

    Mirrors trigger_cian_backfill_run: claim run → create_task → mark_done/failed.
    executor logic lives in tasks/yandex_address_backfill.py which delegates to
    services/yandex_address_backfill.py (backfill_yandex_addresses).

    Returns run_id или None (skip — already running).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            from app.tasks.yandex_address_backfill import run_yandex_address_backfill

            await run_yandex_address_backfill(run_db, run_id=run_id, params=params)
        except Exception:
            logger.exception("scheduler: run_yandex_address_backfill crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered yandex_address_backfill run_id=%d", run_id)
    return run_id


async def trigger_geocode_missing_listings_run(
    db: Session, schedule_row: dict[str, Any]
) -> int | None:
    """Создать scrape_runs + launch run_geocode_missing_listings в asyncio.create_task.

    Nightly backfill listings.lat/lon для всех источников с NULL coords (#1: geom-coverage gap).
    Адрес-dedup batch loop с wall-clock budget (default 30 мин).

    Root cause: scrapers сохраняют listings с lat=lon=NULL — «геокод подтянет cron»,
    но cron геокодирует только deals, не listings. Эта функция замыкает петлю: нигде
    снаружи нет OS cron для listings — только этот планировщик.

    Mirrors trigger_yandex_address_backfill_run: claim run → create_task → mark_done/failed.

    Returns run_id или None (skip — already running).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            from app.tasks.geocode_missing import run_geocode_missing_listings

            await run_geocode_missing_listings(run_db, run_id=run_id, params=params)
        except Exception:
            logger.exception("scheduler: run_geocode_missing_listings crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered geocode_missing_listings run_id=%d", run_id)
    return run_id


async def trigger_rosreestr_dkp_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
    """Создать scrape_runs + launch import_rosreestr_dkp в executor (sync I/O-bound task)."""
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            # import_rosreestr_dkp is synchronous (DB-only, no async HTTP)
            loop = asyncio.get_event_loop()
            await loop.run_in_executor(None, import_rosreestr_dkp, run_db, run_id, params)
        except Exception:
            logger.exception("scheduler: import_rosreestr_dkp crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered rosreestr_dkp_import run_id=%d", run_id)
    return run_id


async def trigger_listing_source_snapshot_run(
    db: Session, schedule_row: dict[str, Any]
) -> int | None:
    """Создать scrape_runs + launch snapshot_listing_sources в executor (sync DB-only task).

    Per-source price-history daily snapshot (#570). Mirror trigger_rosreestr_dkp_run:
    задача синхронная (чистый internal DB-snapshot, БЕЗ внешних HTTP-вызовов / анти-бота),
    поэтому гоняем её в run_in_executor. SAFE to enable — schedule seed enabled=true (079),
    в отличие от avito/yandex sweep (dormant).

    Returns run_id (или None если skip — есть running run).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            # snapshot_listing_sources is synchronous (DB-only, no async HTTP)
            from app.tasks.listing_source_snapshot import snapshot_listing_sources

            loop = asyncio.get_event_loop()
            await loop.run_in_executor(None, snapshot_listing_sources, run_db, run_id)
        except Exception:
            logger.exception("scheduler: snapshot_listing_sources crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered listing_source_snapshot run_id=%d", run_id)
    return run_id


async def trigger_refresh_search_matview_run(
    db: Session, schedule_row: dict[str, Any]
) -> int | None:
    """Создать scrape_runs + запустить REFRESH MATERIALIZED VIEW CONCURRENTLY listings_search_mv.

    listings_search_mv активно читается search_query.py (SELECT FROM listings_search_mv).
    Без периодического refresh матвью устаревает по мере появления новых listings.
    Задача синхронная (DB-only, БЕЗ внешних HTTP-вызовов) — гоняем в run_in_executor
    по образцу trigger_asking_to_sold_ratio_run (#648).

    refresh_search_matview() использует собственное psycopg-соединение с autocommit=True
    (REFRESH CONCURRENTLY не может работать внутри транзакции). scrape_runs lifecycle
    (mark_done/mark_failed) управляется через отдельную SessionLocal.

    SAFE to enable — schedule seed enabled=true (088), pure internal DB op.
    Окно 03:00-04:00 UTC — ночное, до rosreestr_dkp_import (04:00-06:00 UTC).

    Returns run_id (или None если skip — есть running run).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            from app.tasks.refresh_search_matview import refresh_search_matview

            loop = asyncio.get_event_loop()
            # refresh_search_matview() manages its own connection with autocommit=True
            # (required for REFRESH MATERIALIZED VIEW CONCURRENTLY outside a transaction).
            await loop.run_in_executor(None, refresh_search_matview)
            runs_mod.mark_done(run_db, run_id, {})
        except Exception:
            logger.exception("scheduler: refresh_search_matview crashed run_id=%d", run_id)
            runs_mod.mark_failed(run_db, run_id, "refresh_search_matview failed", {})
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered refresh_search_matview run_id=%d", run_id)
    return run_id


async def trigger_deactivate_stale_avito_run(
    db: Session, schedule_row: dict[str, Any]
) -> int | None:
    """Создать scrape_runs + launch deactivate_stale_avito_listings в executor (sync DB-only task).

    Помечает avito-объявления, не виденные >TTL дней, как is_active=false (#759).
    Задача синхронная (чистый internal UPDATE, БЕЗ внешних HTTP-вызовов) — гоняем в
    run_in_executor по образцу trigger_listing_source_snapshot_run. SAFE to enable —
    schedule seed enabled=true (090), pure internal DB op.
    Окно 06:00-07:00 UTC — после avito_city_sweep (02:00-05:00 UTC).

    Returns run_id (или None если skip — есть running run).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            from app.tasks.deactivate_stale_avito import deactivate_stale_avito_listings

            loop = asyncio.get_event_loop()
            await loop.run_in_executor(None, deactivate_stale_avito_listings, run_db, run_id)
        except Exception:
            logger.exception("scheduler: deactivate_stale_avito crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered deactivate_stale_avito run_id=%d", run_id)
    return run_id


async def trigger_sber_index_pull_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
    """Создать scrape_runs + launch run_sber_index_pull в asyncio.create_task.

    Monthly pull (#887): fetches city-level secondary-housing price index from
    sberindex.ru/api/sowa across three dashboards (real_estate_deals,
    residential_real_estate_prices, dinamika-tsen-obyavlenii) and upserts into
    sber_price_index. Pure HTTP+DB, no scraper proxy, no anti-bot.

    Mirrors trigger_yandex_address_backfill_run: claim run → create_task → mark_done/failed
    delegated to tasks/sber_index_pull.py.

    Returns run_id или None (skip — already running).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            from app.tasks.sber_index_pull import run_sber_index_pull

            await run_sber_index_pull(run_db, run_id=run_id, params=params)
        except Exception:
            logger.exception("scheduler: run_sber_index_pull crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered sber_index_pull run_id=%d", run_id)
    return run_id


async def trigger_rosreestr_quarter_poll_run(
    db: Session, schedule_row: dict[str, Any]
) -> int | None:
    """Создать scrape_runs + launch run_rosreestr_quarter_poll в asyncio.create_task.

    Monthly HEAD-check (#888): polls rosreestr.gov.ru for the next quarter's open-data
    ZIP to detect when a new quarter is published. Pure HTTP detect+alert — does NOT
    download the multi-GB dataset or shell out to the loaders. When available=True, a
    clear actionable INFO log is emitted for the operator.

    Mirrors trigger_sber_index_pull_run: claim run → create_task → mark_done/failed
    delegated to tasks/rosreestr_quarter_poll.py.

    Returns run_id или None (skip — already running).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            from app.tasks.rosreestr_quarter_poll import run_rosreestr_quarter_poll

            await run_rosreestr_quarter_poll(run_db, run_id=run_id, params=params)
        except Exception:
            logger.exception("scheduler: run_rosreestr_quarter_poll crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered rosreestr_quarter_poll run_id=%d", run_id)
    return run_id


async def trigger_newbuilding_enrich_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
    """Создать scrape_runs + launch run_newbuilding_enrich в asyncio.create_task (#973).

    Nightly slice of the cian_newbuilding enrichment backfill: resolves cian_zhk_url from
    house_sources.ext_id when missing, fetches the ЖК page (curl_cffi chrome120) and persists
    houses_price_dynamics / house_reliability_checks / house_reviews. This is the standalone
    schedule the inline cian full-sweep path never had — 306 fetchable cian houses on prod,
    only ~3 enriched so far, so the backlog drains over successive nights.

    Mirrors trigger_sber_index_pull_run: claim run → create_task → mark_done/failed delegated
    to tasks/newbuilding_enrich_backfill.run_newbuilding_enrich (which owns the run lifecycle).

    Idempotency: run_newbuilding_enrich → backfill_newbuilding_enrichment skips houses that
    already have BOTH price_dynamics AND reliability rows (force=False) and uses a per-house
    SAVEPOINT, so a re-fire never re-enriches done houses or double-appends on a partial run.
    `limit` (default_params) bounds how many houses one fire fetches — anti-bot politeness;
    the nightly cadence drains the rest. zombie-reap is inherited from the shared loop.

    Returns run_id или None (skip — already running).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            from app.tasks.newbuilding_enrich_backfill import run_newbuilding_enrich

            await run_newbuilding_enrich(run_db, run_id=run_id, params=params)
        except Exception:
            logger.exception("scheduler: run_newbuilding_enrich crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered newbuilding_enrich run_id=%d", run_id)
    return run_id


async def trigger_yandex_newbuilding_sweep_run(
    db: Session, schedule_row: dict[str, Any]
) -> int | None:
    """Создать scrape_runs + launch enrich_yandex_newbuilding_sweep в asyncio.create_task (#974).

    Nightly enrichment sweep: SELECT pending yandex_realty_nb houses → resolve slug
    (BrowserFetcher SERP) → fetch_jk (BrowserFetcher) → UPSERT market.yandex_jk_enrichment.

    Shipped DORMANT (seed 106, enabled=false). Включается оператором вручную:
        UPDATE scrape_schedules SET enabled = true WHERE source = 'yandex_newbuilding_sweep';

    Mirrors trigger_newbuilding_enrich_run: claim run → create_task → mark_done/failed
    делегируется tasks/yandex_newbuilding_sweep.enrich_yandex_newbuilding_sweep.

    Returns run_id или None (skip — already running).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            from app.tasks.yandex_newbuilding_sweep import enrich_yandex_newbuilding_sweep

            result = await enrich_yandex_newbuilding_sweep(
                run_db,
                limit=int(params.get("limit", 5)),
                request_delay_sec=float(params.get("request_delay_sec", 8.0)),
                city=str(params.get("city", "ekaterinburg")),
            )
            runs_mod.mark_done(run_db, run_id, result.to_dict())
        except Exception:
            logger.exception("scheduler: enrich_yandex_newbuilding_sweep crashed run_id=%d", run_id)
            try:
                runs_mod.mark_failed(run_db, run_id, "crashed", {})
            except Exception:
                pass
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered yandex_newbuilding_sweep run_id=%d", run_id)
    return run_id


async def trigger_asking_to_sold_ratio_run(db: Session, schedule_row: dict[str, Any]) -> int | None:
    """Создать scrape_runs + launch recompute_asking_to_sold_ratios в executor (sync DB-only task).

    Daily TRUE-MIRROR refresh асking→sold коэффициентов (#648 Stage 4): DELETE WHERE
    district='' + re-derive INSERT (та же 080-derivation) в одной транзакции, чтобы ratio
    не устаревал по мере ночного импорта ДКП. Mirror trigger_listing_source_snapshot_run:
    задача синхронная (чистый internal DB re-derive, БЕЗ внешних HTTP-вызовов / анти-бота),
    поэтому гоняем её в run_in_executor. SAFE to enable — schedule seed enabled=true (082),
    в отличие от avito/yandex sweep (dormant).

    Returns run_id (или None если skip — есть running run).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            # recompute_asking_to_sold_ratios is synchronous (DB-only, no async HTTP)
            from app.tasks.asking_to_sold_ratio import recompute_asking_to_sold_ratios

            loop = asyncio.get_event_loop()
            await loop.run_in_executor(None, recompute_asking_to_sold_ratios, run_db, run_id)
        except Exception:
            logger.exception("scheduler: recompute_asking_to_sold_ratios crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered asking_to_sold_ratio_refresh run_id=%d", run_id)
    return run_id


def import_rosreestr_dkp(
    db: Session,
    run_id: int,
    params: dict[str, Any],
) -> None:
    """Import ДКП-сделок из gendesign rosreestr_deals через postgres_fdw.

    Python-порт import-rosreestr.sh (Variant C из #563).

    Источник: foreign table gendesign_rosreestr_deals (создана в migration 072).
    SERVER gendesign_remote настроен в 060_postgres_fdw_extension.sql.
    USER MAPPING создаётся при startup в core/fdw.py (tradein_fdw_reader).

    Фильтры (совпадают с import-rosreestr.sh + Fix_Rosreestr_Dkp_Filter_May24):
      - region_code = 66 (Свердловская область)
      - city ILIKE '%катеринбург%'
      - realestate_type_code = '002001003000' (квартира)
      - area BETWEEN 18 AND 200
      - deal_price BETWEEN 1000000 AND 100000000
      - street IS NOT NULL AND trim(street) != ''
      - doc_type = 'ДКП'  (только вторичка — #549 / Fix_Rosreestr_Dkp_Filter_May24)
      - period_start_date >= since  (default '2024-01-01')

    dedup_hash: 'ros:dkp:' || id — плоский натуральный ключ (инъективный, без коллизий,
      human-readable). До #576 здесь был md5('ros:dkp:' || id); миграция 077 конвертировала
      существующие строки. source_id хранит исходный rosreestr id (дедуп переустанавливаем).
    Rooms: выводятся из площади (Росреестр не отдаёт кол-во комнат).

    Batch-процессинг: читаем из FDW батчами по batch_size через cursor-based пагинацию
    (WHERE id > last_id ORDER BY id). Heartbeat обновляется каждый батч (= checkpoint).
    SAVEPOINT per row — один сбойный row не откатывает батч.

    Координаты: NULL после импорта — геокодинг остаётся follow-up (geocode-deals).
    TODO (follow-up): запустить geocode backfill после import.

    Cleanup: удаляет legacy строки address='Екатеринбург, реальная сделка' (pre-#549).
    """
    since: str = str(params.get("since", "2024-01-01"))
    batch_size: int = int(params.get("batch_size", 2000))

    counters: dict[str, int] = {
        "rows_fetched": 0,
        "rows_inserted": 0,
        "rows_skipped": 0,
        "batches_done": 0,
    }

    # Cleanup legacy synthetic rows (pre-#549, idempotent)
    try:
        deleted = db.execute(
            text("DELETE FROM deals WHERE address = CAST(:addr AS text) RETURNING id"),
            {"addr": "Екатеринбург, реальная сделка"},
        ).fetchall()
        db.commit()
        if deleted:
            logger.info(
                "rosreestr_dkp_import run_id=%d: removed %d legacy synthetic rows",
                run_id,
                len(deleted),
            )
    except Exception as exc:
        logger.warning(
            "rosreestr_dkp_import run_id=%d: cleanup failed (non-fatal): %s", run_id, exc
        )
        db.rollback()

    last_id: int = 0
    total_batches = 0

    try:
        while True:
            if runs_mod.is_cancelled(db, run_id):
                logger.info("rosreestr_dkp_import run_id=%d: cancelled by user", run_id)
                runs_mod.mark_cancelled(db, run_id)
                return

            # Cursor-based pagination via foreign table gendesign_rosreestr_deals.
            # FDW pushes WHERE + ORDER BY + LIMIT to gendesign-postgres automatically
            # (postgres_fdw 'use_remote_estimate' is off by default but clause push-down
            # still happens for simple predicates on the remote side).
            batch_rows = (
                db.execute(
                    text("""
                    SELECT
                        id,
                        id                                       AS source_id_src,
                        'ros:dkp:' || CAST(id AS text)           AS dedup_hash,
                        'Екатеринбург, ' || trim(street)         AS address,
                        CASE
                            WHEN area < 30 THEN 0
                            WHEN area < 44 THEN 1
                            WHEN area < 62 THEN 2
                            WHEN area < 85 THEN 3
                            ELSE 4
                        END                                      AS rooms,
                        round(area, 2)                           AS area_m2,
                        LEAST(
                            NULLIF(
                                -- #1525: '-?[0-9]+' сохраняет знак минус, иначе цоколь/подвал
                                -- '-1' импортируется как 1. Для '5/9' по-прежнему берётся 5.
                                substring(floor FROM '-?[0-9]+'), ''
                            )::int,
                            100
                        )                                        AS floor_num,
                        year_build                               AS year_built,
                        round(deal_price)::bigint                AS price_rub,
                        round(price_per_sqm)::int                AS price_per_m2,
                        period_start_date                        AS deal_date
                    FROM gendesign_rosreestr_deals
                    WHERE region_code = 66
                      AND city ILIKE '%катеринбург%'
                      AND realestate_type_code = '002001003000'
                      AND area BETWEEN 18 AND 200
                      AND deal_price BETWEEN 1000000 AND 100000000
                      AND street IS NOT NULL AND trim(street) <> ''
                      AND doc_type = 'ДКП'
                      AND period_start_date >= CAST(:since AS date)
                      AND id > CAST(:last_id AS bigint)
                    ORDER BY id
                    LIMIT CAST(:batch_size AS int)
                """),
                    {"since": since, "last_id": last_id, "batch_size": batch_size},
                )
                .mappings()
                .all()
            )

            if not batch_rows:
                break

            total_batches += 1
            batch_inserted = 0
            batch_skipped = 0
            batch_max_id = last_id

            for row in batch_rows:
                row_id: int = int(row["id"])
                if row_id > batch_max_id:
                    batch_max_id = row_id

                try:
                    with db.begin_nested():  # SAVEPOINT per row
                        inserted = db.execute(
                            text("""
                                INSERT INTO deals (
                                    source, dedup_hash, source_id, address, rooms, area_m2,
                                    floor, year_built, price_rub, price_per_m2, deal_date
                                )
                                VALUES (
                                    'rosreestr',
                                    CAST(:dedup_hash AS text),
                                    CAST(:source_id AS text),
                                    CAST(:address AS text),
                                    CAST(:rooms AS int),
                                    CAST(:area_m2 AS numeric),
                                    CAST(:floor_num AS int),
                                    CAST(:year_built AS int),
                                    CAST(:price_rub AS bigint),
                                    CAST(:price_per_m2 AS int),
                                    CAST(:deal_date AS date)
                                )
                                ON CONFLICT (dedup_hash) DO NOTHING
                                RETURNING id
                            """),
                            {
                                "dedup_hash": row["dedup_hash"],
                                "source_id": str(row["source_id_src"]),
                                "address": row["address"],
                                "rooms": row["rooms"],
                                "area_m2": row["area_m2"],
                                "floor_num": row["floor_num"],
                                "year_built": row["year_built"],
                                "price_rub": row["price_rub"],
                                "price_per_m2": row["price_per_m2"],
                                "deal_date": row["deal_date"],
                            },
                        ).fetchone()
                        if inserted is not None:
                            batch_inserted += 1
                        else:
                            batch_skipped += 1
                except Exception as exc:
                    logger.warning(
                        "rosreestr_dkp_import run_id=%d: row id=%d INSERT failed: %s",
                        run_id,
                        row_id,
                        exc,
                    )
                    batch_skipped += 1

            last_id = batch_max_id
            db.commit()

            counters["rows_fetched"] += len(batch_rows)
            counters["rows_inserted"] += batch_inserted
            counters["rows_skipped"] += batch_skipped
            counters["batches_done"] = total_batches
            counters["last_id"] = last_id  # type: ignore[assignment]

            # Heartbeat = checkpoint: allows zombie detection + resume visibility
            runs_mod.update_heartbeat(db, run_id, counters)
            logger.info(
                "rosreestr_dkp_import run_id=%d: batch=%d fetched=%d "
                "inserted=%d skipped=%d last_id=%d",
                run_id,
                total_batches,
                len(batch_rows),
                batch_inserted,
                batch_skipped,
                last_id,
            )

            if len(batch_rows) < batch_size:
                break  # Last partial batch — no more rows

        runs_mod.mark_done(db, run_id, counters)
        logger.info(
            "rosreestr_dkp_import run_id=%d done: "
            "total_fetched=%d inserted=%d skipped=%d batches=%d",
            run_id,
            counters["rows_fetched"],
            counters["rows_inserted"],
            counters["rows_skipped"],
            total_batches,
        )

    except Exception as exc:
        logger.exception("rosreestr_dkp_import run_id=%d failed at last_id=%d", run_id, last_id)
        runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters)
        raise


async def trigger_avito_detail_backfill_run(
    db: Session, schedule_row: dict[str, Any]
) -> int | None:
    """Create scrape_run + launch run_avito_detail_backfill in asyncio.create_task.

    Nightly backfill detail_enriched_at for legacy avito listings (#1551).
    ~15243/15917 avito listings have detail_enriched_at IS NULL -- never enriched
    by city_sweep ENRICH_DETAIL (only top-N proximate per anchor are touched).

    Mirrors trigger_geocode_missing_listings_run: claim run -> create_task ->
    mark_done/failed delegated to tasks/avito_detail_backfill.run_avito_detail_backfill.

    Returns run_id or None (skip -- already running).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            from app.tasks.avito_detail_backfill import run_avito_detail_backfill

            await run_avito_detail_backfill(run_db, run_id=run_id, params=params)
        except Exception:
            logger.exception("scheduler: run_avito_detail_backfill crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered avito_detail_backfill run_id=%d", run_id)
    return run_id


async def trigger_yandex_detail_backfill_run(
    db: Session, schedule_row: dict[str, Any]
) -> int | None:
    """Create scrape_run + launch run_yandex_detail_backfill in asyncio.create_task.

    Nightly backfill detail_enriched_at for legacy yandex listings (#1553).
    ~3952 yandex listings have detail_enriched_at IS NULL -- SERP-only data,
    never enriched by YandexDetailScraper.

    Mirrors trigger_avito_detail_backfill_run: claim run -> create_task ->
    mark_done/failed delegated to tasks/yandex_detail_backfill.run_yandex_detail_backfill.

    Returns run_id or None (skip -- already running).
    """
    run_id = _claim_run(db, schedule_row)
    if run_id is None:
        return None

    params = schedule_row.get("default_params") or {}

    async def _run() -> None:
        run_db = SessionLocal()
        try:
            from app.tasks.yandex_detail_backfill import run_yandex_detail_backfill

            await run_yandex_detail_backfill(run_db, run_id=run_id, params=params)
        except Exception:
            logger.exception("scheduler: run_yandex_detail_backfill crashed run_id=%d", run_id)
        finally:
            run_db.close()

    task = asyncio.create_task(_run())
    task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None)
    logger.info("scheduler: triggered yandex_detail_backfill run_id=%d", run_id)
    return run_id


def get_due_schedules(db: Session) -> list[dict[str, Any]]:
    """SELECT scrape_schedules WHERE enabled AND (next_run_at IS NULL OR next_run_at <= NOW())."""
    rows = (
        db.execute(
            text(
                """
            SELECT id, source, enabled, window_start_hour, window_end_hour,
                   default_params, last_run_id, last_run_at, next_run_at
            FROM scrape_schedules
            WHERE enabled = true
              AND (next_run_at IS NULL OR next_run_at <= NOW())
            """
            ),
        )
        .mappings()
        .all()
    )
    return [dict(r) for r in rows]


async def scheduler_loop() -> None:
    """Бесконечный async loop — tick каждые SCHEDULER_TICK_SEC секунд."""
    logger.info("scheduler: started (tick=%ds)", SCHEDULER_TICK_SEC)
    # Initial sleep 30s чтобы дать FastAPI startup завершиться
    await asyncio.sleep(30)
    while True:
        try:
            db = SessionLocal()
            try:
                # Reap zombies first
                reap_zombies(db)
                # Process due schedules
                due = get_due_schedules(db)
                for sch in due:
                    source = sch["source"]
                    if source == "avito_city_sweep":
                        await trigger_avito_city_sweep_run(db, sch)
                    elif source == "yandex_city_sweep":
                        await trigger_yandex_city_sweep_run(db, sch)
                    elif source == "n1_city_sweep":
                        await trigger_n1_city_sweep_run(db, sch)
                    elif source == "cian_history_backfill":
                        await trigger_cian_backfill_run(db, sch)
                    elif source == "cian_city_sweep":
                        await trigger_cian_city_sweep_run(db, sch)
                    elif source == "rosreestr_dkp_import":
                        await trigger_rosreestr_dkp_run(db, sch)
                    elif source == "listing_source_snapshot":
                        await trigger_listing_source_snapshot_run(db, sch)
                    elif source == "asking_to_sold_ratio_refresh":
                        await trigger_asking_to_sold_ratio_run(db, sch)
                    elif source == "refresh_search_matview":
                        await trigger_refresh_search_matview_run(db, sch)
                    elif source == "yandex_address_backfill":
                        await trigger_yandex_address_backfill_run(db, sch)
                    elif source == "deactivate_stale_avito":
                        await trigger_deactivate_stale_avito_run(db, sch)
                    elif source == "sber_index_pull":
                        await trigger_sber_index_pull_run(db, sch)
                    elif source == "rosreestr_quarter_poll":
                        await trigger_rosreestr_quarter_poll_run(db, sch)
                    elif source == "newbuilding_enrich":
                        await trigger_newbuilding_enrich_run(db, sch)
                    elif source == "yandex_newbuilding_sweep":
                        await trigger_yandex_newbuilding_sweep_run(db, sch)
                    elif source == "geocode_missing_listings":
                        await trigger_geocode_missing_listings_run(db, sch)
                    elif source == "avito_detail_backfill":
                        await trigger_avito_detail_backfill_run(db, sch)
                    elif source == "yandex_detail_backfill":
                        await trigger_yandex_detail_backfill_run(db, sch)
                    else:
                        logger.warning("scheduler: unknown source=%s, skip", source)
            finally:
                db.close()
        except Exception:
            logger.exception("scheduler: tick failed")
        await asyncio.sleep(SCHEDULER_TICK_SEC)