gendesign/tradein-mvp/backend/tests/test_house_imv_backfill_scheduler.py

"""Tests for wiring house_imv_backfill as an in-app scheduler source (#854).

The bulk IMV backfill service (services/house_imv_backfill.backfill_house_imv) is
already built + covered elsewhere; this gate covers ONLY the scheduler wiring added
by #854:
  - trigger_house_imv_backfill_run shape (_claim_run → asyncio.create_task → fresh
    SessionLocal → mark_done/mark_failed lifecycle → add_done_callback / RUF006),
  - dispatch branch in scheduler_loop routes source=="house_imv_backfill",
  - migration 132 seeds an idempotent scrape_schedules row with default_params.

Convention mirrors tests/tasks/test_cadastral_geo_match.py: hermetic static analysis
(inspect.getsource + migration file read) plus a fake-async behavioural test driving
the trigger's lifecycle without a live Postgres.
"""

from __future__ import annotations

import asyncio
import inspect
import os
import re
from pathlib import Path
from typing import Any

import pytest

# settings needs a DSN at import (same dance as the sibling tests); these are static/fake-db.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from app.services import scheduler

_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql"
_MIGRATION_132 = _SQL_DIR / "132_scrape_schedules_seed_house_imv.sql"

_TRIGGER_SRC = inspect.getsource(scheduler.trigger_house_imv_backfill_run)


# ── Scheduler wiring ──────────────────────────────────────────────────────────


def test_scheduler_has_trigger_and_dispatch() -> None:
    assert hasattr(scheduler, "trigger_house_imv_backfill_run")
    loop_src = inspect.getsource(scheduler.scheduler_loop)
    assert 'source == "house_imv_backfill"' in loop_src
    assert "trigger_house_imv_backfill_run(db, sch)" in loop_src


def test_trigger_mirrors_backfill_sibling_pattern() -> None:
    """_claim_run gate → fresh SessionLocal → backfill_house_imv → add_done_callback."""
    assert "_claim_run(db, schedule_row)" in _TRIGGER_SRC
    assert "if run_id is None:" in _TRIGGER_SRC
    # fresh session inside the spawned task (not the scheduler-tick session).
    assert "run_db = SessionLocal()" in _TRIGGER_SRC
    assert "backfill_house_imv" in _TRIGGER_SRC
    # async service → bare await (NOT run_in_executor, unlike the sync DB-only triggers).
    assert "run_in_executor" not in _TRIGGER_SRC
    # RUF006: keep a reference + done-callback so the task is not GC'd mid-flight.
    assert "task = asyncio.create_task(_run())" in _TRIGGER_SRC
    assert "task.add_done_callback(" in _TRIGGER_SRC
    assert "run_db.close()" in _TRIGGER_SRC


def test_trigger_owns_run_lifecycle() -> None:
    """No tasks/*-wrapper for this service → the trigger finalises scrape_runs itself."""
    assert "runs_mod.mark_done(" in _TRIGGER_SRC
    assert "runs_mod.mark_failed(" in _TRIGGER_SRC
    assert "logger.exception" in _TRIGGER_SRC
    # heartbeat callback keeps the long IMV run alive vs reap_zombies (#1363).
    assert "heartbeat=" in _TRIGGER_SRC
    assert "update_heartbeat" in _TRIGGER_SRC


def test_trigger_reads_default_params() -> None:
    """default_params drive batch_size / request_delay_sec / only_status with safe defaults."""
    assert 'params.get("batch_size"' in _TRIGGER_SRC
    assert 'params.get("request_delay_sec"' in _TRIGGER_SRC
    assert 'params.get("only_status"' in _TRIGGER_SRC


# ── Migration content ─────────────────────────────────────────────────────────


def test_migration_132_seeds_schedule_idempotent() -> None:
    sql = _MIGRATION_132.read_text(encoding="utf-8")
    assert "INSERT INTO scrape_schedules" in sql
    assert "'house_imv_backfill'" in sql
    assert "ON CONFLICT (source) DO NOTHING" in sql
    assert "BEGIN;" in sql and "COMMIT;" in sql


def test_migration_132_next_run_not_null_tomorrow() -> None:
    """next_run_at NOT NULL = tomorrow (avoid deploy-time fire), per seed convention."""
    sql = _MIGRATION_132.read_text(encoding="utf-8")
    assert "CURRENT_DATE + INTERVAL '1 day'" in sql
    assert "make_interval(hours => 15)" in sql


def test_migration_132_default_params() -> None:
    sql = _MIGRATION_132.read_text(encoding="utf-8")
    assert '"batch_size": 50' in sql
    assert '"request_delay_sec": 5' in sql
    assert '"only_status": "pending"' in sql


def test_migration_132_no_psycopg_v3_colon_colon_cast() -> None:
    """psycopg v3: never :param::type — must use CAST(:param AS type)."""
    sql = _MIGRATION_132.read_text(encoding="utf-8")
    assert not re.search(r":\w+::", sql)


def test_migration_132_matches_seed_column_set() -> None:
    """Exact column set of existing scrape_schedules seeds (source..default_params)."""
    sql = _MIGRATION_132.read_text(encoding="utf-8")
    for col in (
        "source",
        "enabled",
        "window_start_hour",
        "window_end_hour",
        "next_run_at",
        "default_params",
    ):
        assert col in sql


# ── Fake-async behavioural: trigger lifecycle without Postgres ────────────────


class _FakeDB:
    """Minimal Session stand-in: only needs to be a distinct object + close()."""

    def __init__(self) -> None:
        self.closed = 0

    def close(self) -> None:
        self.closed += 1


def test_trigger_skips_when_claim_returns_none(monkeypatch: pytest.MonkeyPatch) -> None:
    """If _claim_run finds a running run (None) → trigger returns None, spawns nothing."""
    monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: None)

    async def _go() -> int | None:
        return await scheduler.trigger_house_imv_backfill_run(
            _FakeDB(),  # type: ignore[arg-type]
            {"source": "house_imv_backfill", "default_params": {}},
        )

    assert asyncio.run(_go()) is None


def test_trigger_claims_runs_backfill_and_marks_done(monkeypatch: pytest.MonkeyPatch) -> None:
    """Happy path: claim run → backfill_house_imv awaited with params → mark_done(counters)."""
    captured: dict[str, Any] = {}
    marked: dict[str, Any] = {}

    monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: 77)
    monkeypatch.setattr(scheduler, "SessionLocal", _FakeDB)
    monkeypatch.setattr(scheduler.runs_mod, "update_heartbeat", lambda *a, **k: None)
    monkeypatch.setattr(
        scheduler.runs_mod,
        "mark_done",
        lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)),
    )
    monkeypatch.setattr(scheduler.runs_mod, "mark_failed", lambda *a, **k: None)

    class _Result:
        checked, saved, skipped, errors, duration_sec = 10, 6, 2, 2, 42.0

    async def _fake_backfill(_db: Any, **kwargs: Any) -> _Result:
        captured.update(kwargs)
        return _Result()

    import app.services.house_imv_backfill as _svc

    monkeypatch.setattr(_svc, "backfill_house_imv", _fake_backfill)

    async def _go() -> int | None:
        run_id = await scheduler.trigger_house_imv_backfill_run(
            _FakeDB(),  # type: ignore[arg-type]
            {
                "source": "house_imv_backfill",
                "default_params": {
                    "batch_size": 25,
                    "request_delay_sec": 3,
                    "only_status": "transient_error",
                },
            },
        )
        # let the spawned _run() task finish.
        await asyncio.sleep(0)
        await asyncio.sleep(0)
        return run_id

    run_id = asyncio.run(_go())
    assert run_id == 77
    assert captured["batch_size"] == 25
    assert captured["request_delay_sec"] == 3.0
    assert captured["only_status"] == "transient_error"
    assert callable(captured["heartbeat"])
    assert marked["run_id"] == 77
    assert marked["counters"]["checked"] == 10
    assert marked["counters"]["saved"] == 6


def test_trigger_marks_failed_on_backfill_crash(monkeypatch: pytest.MonkeyPatch) -> None:
    """If backfill_house_imv raises → trigger marks the run failed (does not bubble)."""
    failed: dict[str, Any] = {}

    monkeypatch.setattr(scheduler, "_claim_run", lambda _db, _row: 88)
    monkeypatch.setattr(scheduler, "SessionLocal", _FakeDB)
    monkeypatch.setattr(scheduler.runs_mod, "update_heartbeat", lambda *a, **k: None)
    monkeypatch.setattr(scheduler.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(
        scheduler.runs_mod,
        "mark_failed",
        lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err),
    )

    async def _boom(_db: Any, **kwargs: Any) -> None:
        raise RuntimeError("imv 403")

    import app.services.house_imv_backfill as _svc

    monkeypatch.setattr(_svc, "backfill_house_imv", _boom)

    async def _go() -> None:
        await scheduler.trigger_house_imv_backfill_run(
            _FakeDB(),  # type: ignore[arg-type]
            {"source": "house_imv_backfill", "default_params": {}},
        )
        await asyncio.sleep(0)
        await asyncio.sleep(0)

    asyncio.run(_go())
    assert failed["run_id"] == 88
    assert "imv 403" in failed["err"]