"""Tests for wiring house_imv_backfill as an in-app scheduler source (#854). The bulk IMV backfill service (services/house_imv_backfill.backfill_house_imv) is already built + covered elsewhere; this gate covers ONLY the scheduler wiring added by #854: - trigger_house_imv_backfill_run shape (_claim_run → asyncio.create_task → fresh SessionLocal → mark_done/mark_failed lifecycle → add_done_callback / RUF006), - dispatch branch in scheduler_loop routes source=="house_imv_backfill", - migration 132 seeds an idempotent scrape_schedules row with default_params. Convention mirrors tests/tasks/test_cadastral_geo_match.py: hermetic static analysis (inspect.getsource + migration file read) plus a fake-async behavioural test driving the trigger's lifecycle without a live Postgres. """ from __future__ import annotations import asyncio import inspect import os import re from pathlib import Path from typing import Any import pytest # settings needs a DSN at import (same dance as the sibling tests); these are static/fake-db. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from app.services import scheduler _SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql" _MIGRATION_132 = _SQL_DIR / "132_scrape_schedules_seed_house_imv.sql" _TRIGGER_SRC = inspect.getsource(scheduler.trigger_house_imv_backfill_run) # ── Scheduler wiring ────────────────────────────────────────────────────────── def test_scheduler_has_trigger_and_dispatch() -> None: assert hasattr(scheduler, "trigger_house_imv_backfill_run") loop_src = inspect.getsource(scheduler.scheduler_loop) assert 'source == "house_imv_backfill"' in loop_src assert "trigger_house_imv_backfill_run(db, sch)" in loop_src def test_trigger_mirrors_backfill_sibling_pattern() -> None: """_claim_run gate → fresh SessionLocal → backfill_house_imv → add_done_callback.""" assert "_claim_run(db, schedule_row)" in _TRIGGER_SRC assert "if run_id is None:" in _TRIGGER_SRC # fresh session inside the spawned task (not the scheduler-tick session). assert "run_db = SessionLocal()" in _TRIGGER_SRC assert "backfill_house_imv" in _TRIGGER_SRC # async service → bare await (NOT run_in_executor, unlike the sync DB-only triggers). assert "run_in_executor" not in _TRIGGER_SRC # #1182 P2: detached spawn через _spawn_tracked — strong-ref в registry (RUF006 + # graceful SIGTERM-drain), done-callback ретривит exception. Заменил прежний # `task = asyncio.create_task(_run()); task.add_done_callback(...)`. assert "_spawn_tracked(_run())" in _TRIGGER_SRC assert "run_db.close()" in _TRIGGER_SRC def test_trigger_owns_run_lifecycle() -> None: """No tasks/*-wrapper for this service → the trigger finalises scrape_runs itself.""" assert "runs_mod.mark_done(" in _TRIGGER_SRC assert "runs_mod.mark_failed(" in _TRIGGER_SRC assert "logger.exception" in _TRIGGER_SRC # heartbeat callback keeps the long IMV run alive vs reap_zombies (#1363). assert "heartbeat=" in _TRIGGER_SRC assert "update_heartbeat" in _TRIGGER_SRC def test_trigger_reads_default_params() -> None: """default_params drive batch_size / request_delay_sec / only_status with safe defaults.""" assert 'params.get("batch_size"' in _TRIGGER_SRC assert 'params.get("request_delay_sec"' in _TRIGGER_SRC assert 'params.get("only_status"' in _TRIGGER_SRC # ── Migration content ───────────────────────────────────────────────────────── def test_migration_132_seeds_schedule_idempotent() -> None: sql = _MIGRATION_132.read_text(encoding="utf-8") assert "INSERT INTO scrape_schedules" in sql assert "'house_imv_backfill'" in sql assert "ON CONFLICT (source) DO NOTHING" in sql assert "BEGIN;" in sql and "COMMIT;" in sql def test_migration_132_next_run_not_null_tomorrow() -> None: """next_run_at NOT NULL = tomorrow (avoid deploy-time fire), per seed convention.""" sql = _MIGRATION_132.read_text(encoding="utf-8") assert "CURRENT_DATE + INTERVAL '1 day'" in sql assert "make_interval(hours => 15)" in sql def test_migration_132_default_params() -> None: sql = _MIGRATION_132.read_text(encoding="utf-8") assert '"batch_size": 50' in sql assert '"request_delay_sec": 5' in sql assert '"only_status": "pending"' in sql def test_migration_132_no_psycopg_v3_colon_colon_cast() -> None: """psycopg v3: never :param::type — must use CAST(:param AS type).""" sql = _MIGRATION_132.read_text(encoding="utf-8") assert not re.search(r":\w+::", sql) def test_migration_132_matches_seed_column_set() -> None: """Exact column set of existing scrape_schedules seeds (source..default_params).""" sql = _MIGRATION_132.read_text(encoding="utf-8") for col in ( "source", "enabled", "window_start_hour", "window_end_hour", "next_run_at", "default_params", ): assert col in sql # ── Fake-async behavioural: trigger lifecycle without Postgres ──────────────── class _FakeDB: """Minimal Session stand-in: only needs to be a distinct object + close().""" def __init__(self) -> None: self.closed = 0 def close(self) -> None: self.closed += 1 def test_trigger_skips_when_claim_returns_none(monkeypatch: pytest.MonkeyPatch) -> None: """If _claim_run finds a running run (None) → trigger returns None, spawns nothing.""" monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: None) async def _go() -> int | None: return await scheduler.trigger_house_imv_backfill_run( _FakeDB(), # type: ignore[arg-type] {"source": "house_imv_backfill", "default_params": {}}, ) assert asyncio.run(_go()) is None def test_trigger_claims_runs_backfill_and_marks_done(monkeypatch: pytest.MonkeyPatch) -> None: """Happy path: claim run → backfill_house_imv awaited with params → mark_done(counters).""" captured: dict[str, Any] = {} marked: dict[str, Any] = {} monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: 77) monkeypatch.setattr(scheduler, "SessionLocal", _FakeDB) monkeypatch.setattr(scheduler.runs_mod, "update_heartbeat", lambda *a, **k: None) monkeypatch.setattr( scheduler.runs_mod, "mark_done", lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)), ) monkeypatch.setattr(scheduler.runs_mod, "mark_failed", lambda *a, **k: None) class _Result: checked, saved, skipped, errors, duration_sec = 10, 6, 2, 2, 42.0 async def _fake_backfill(_db: Any, **kwargs: Any) -> _Result: captured.update(kwargs) return _Result() import app.services.house_imv_backfill as _svc monkeypatch.setattr(_svc, "backfill_house_imv", _fake_backfill) async def _go() -> int | None: run_id = await scheduler.trigger_house_imv_backfill_run( _FakeDB(), # type: ignore[arg-type] { "source": "house_imv_backfill", "default_params": { "batch_size": 25, "request_delay_sec": 3, "only_status": "transient_error", }, }, ) # let the spawned _run() task finish. await asyncio.sleep(0) await asyncio.sleep(0) return run_id run_id = asyncio.run(_go()) assert run_id == 77 assert captured["batch_size"] == 25 assert captured["request_delay_sec"] == 3.0 assert captured["only_status"] == "transient_error" assert callable(captured["heartbeat"]) assert marked["run_id"] == 77 assert marked["counters"]["checked"] == 10 assert marked["counters"]["saved"] == 6 def test_trigger_marks_failed_on_backfill_crash(monkeypatch: pytest.MonkeyPatch) -> None: """If backfill_house_imv raises → trigger marks the run failed (does not bubble).""" failed: dict[str, Any] = {} monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: 88) monkeypatch.setattr(scheduler, "SessionLocal", _FakeDB) monkeypatch.setattr(scheduler.runs_mod, "update_heartbeat", lambda *a, **k: None) monkeypatch.setattr(scheduler.runs_mod, "mark_done", lambda *a, **k: None) monkeypatch.setattr( scheduler.runs_mod, "mark_failed", lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err), ) async def _boom(_db: Any, **kwargs: Any) -> None: raise RuntimeError("imv 403") import app.services.house_imv_backfill as _svc monkeypatch.setattr(_svc, "backfill_house_imv", _boom) async def _go() -> None: await scheduler.trigger_house_imv_backfill_run( _FakeDB(), # type: ignore[arg-type] {"source": "house_imv_backfill", "default_params": {}}, ) await asyncio.sleep(0) await asyncio.sleep(0) asyncio.run(_go()) assert failed["run_id"] == 88 assert "imv 403" in failed["err"]