gendesign/tradein-mvp/backend/tests/test_listing_source_snapshot.py

"""Tests for the per-source listing price-history snapshot writer (#570).

Writer (snapshot_listing_sources) is SQL-heavy, so most assertions are static: we read
the emitted SQL via inspect/text-attr and check the upsert + event-diff CTE shape, table/
column names, and the psycopg-v3 cast discipline (no :param::type). We also assert the
migration 079 contents.

Plus one cheap behavioural test: a fake db (monkeypatched .execute) drives the counter
logic without a real Postgres.

Static style mirrors tests/test_rosreestr_dedup_key.py / test_yandex_city_sweep.py.
"""

import inspect
import os
import re
from pathlib import Path
from typing import Any

import pytest

# Importing app.services.scheduler / app.tasks pulls app.core.config.Settings → needs
# DATABASE_URL. Stub it BEFORE app imports (as in test_scheduler.py) — these tests are
# static / fake-db; no live database is touched.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from app.tasks import listing_source_snapshot as snap_mod

_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql"
_MIGRATION_079 = _SQL_DIR / "079_listing_source_history.sql"

# Emitted SQL text (SQLAlchemy text() clause → .text gives the raw string).
_SNAPSHOT_SQL = str(snap_mod._SNAPSHOT_SQL.text)
_EVENT_DIFF_SQL = str(snap_mod._EVENT_DIFF_SQL.text)
_ALL_SQL = _SNAPSHOT_SQL + "\n" + _EVENT_DIFF_SQL
_WRITER_SRC = inspect.getsource(snap_mod.snapshot_listing_sources)


# ── Snapshot upsert SQL ───────────────────────────────────────────────────────


def test_snapshot_targets_correct_table_and_columns() -> None:
    assert "INSERT INTO listing_source_snapshots" in _SNAPSHOT_SQL
    for col in (
        "listing_source_id",
        "snapshot_date",
        "price_rub",
        "is_active",
        "last_seen_at",
        "payload_hash",
        "observed_at",
        "run_id",
    ):
        assert col in _SNAPSHOT_SQL, f"snapshot SQL missing column {col!r}"
    assert "FROM listing_sources" in _SNAPSHOT_SQL
    assert "CURRENT_DATE" in _SNAPSHOT_SQL


def test_snapshot_is_upsert_last_write_wins() -> None:
    """ON CONFLICT (listing_source_id, snapshot_date) DO UPDATE — last-write-wins for the day."""
    assert "ON CONFLICT (listing_source_id, snapshot_date) DO UPDATE" in _SNAPSHOT_SQL
    # Collapse runs of whitespace so column-alignment spacing doesn't matter.
    flat = re.sub(r"\s+", " ", _SNAPSHOT_SQL)
    for col in ("price_rub", "is_active", "last_seen_at", "payload_hash", "observed_at", "run_id"):
        assert f"{col} = EXCLUDED.{col}" in flat, f"upsert SET missing {col}"


def test_snapshot_derives_is_active_and_payload_hash() -> None:
    # is_active derived from freshness window; payload_hash = md5(raw_payload::text).
    assert "make_interval(days => :freshness_days)" in _SNAPSHOT_SQL
    assert "AS is_active" in _SNAPSHOT_SQL
    assert "md5(raw_payload::text)" in _SNAPSHOT_SQL


# ── Event-diff CTE SQL ────────────────────────────────────────────────────────


def test_event_diff_is_set_based_cte_not_python_loop() -> None:
    """Event diff is one set-based INSERT … SELECT over a CTE — never a row-by-row loop."""
    assert "WITH today AS" in _EVENT_DIFF_SQL
    assert "prior AS" in _EVENT_DIFF_SQL
    assert "DISTINCT ON (listing_source_id)" in _EVENT_DIFF_SQL
    assert "INSERT INTO listing_source_events" in _EVENT_DIFF_SQL
    # Prior = most-recent snapshot strictly before today.
    assert "snapshot_date < CURRENT_DATE" in _EVENT_DIFF_SQL
    assert "snapshot_date = CURRENT_DATE" in _EVENT_DIFF_SQL
    assert "ORDER BY listing_source_id, snapshot_date DESC" in _EVENT_DIFF_SQL
    # No Python iteration over rows in the writer body (set-based only).
    body = _WRITER_SRC.split('"""', 2)[-1]
    assert "for " not in body, "writer must be set-based — no Python row loop"


def test_event_diff_emits_price_change_with_diff_percent() -> None:
    assert "'price_change'" in _EVENT_DIFF_SQL
    # diff_percent = (new-old)/old*100.
    assert "(t.price_rub - p.price_rub)" in _EVENT_DIFF_SQL
    assert "/ p.price_rub * 100" in _EVENT_DIFF_SQL
    # Only when the price actually changed and old is a usable denominator.
    assert "t.price_rub <> p.price_rub" in _EVENT_DIFF_SQL
    assert "p.price_rub <> 0" in _EVENT_DIFF_SQL
    assert "p.price_rub IS NOT NULL" in _EVENT_DIFF_SQL


def test_event_diff_is_idempotent_on_conflict() -> None:
    assert "ON CONFLICT (listing_source_id, change_time, event_type) DO NOTHING" in _EVENT_DIFF_SQL


# ── psycopg v3 cast discipline ────────────────────────────────────────────────


def test_sql_uses_psycopg_v3_casts_not_double_colon() -> None:
    """psycopg v3: bind params via CAST(:x AS type), never :x::type.

    Literal ::-casts on columns (raw_payload::text, ::numeric) are fine — the ban is
    only on :param::type, which psycopg v3 breaks.
    """
    # No bind-param (:name) followed by ::type anywhere in the emitted SQL.
    assert not re.search(r":\w+::", _ALL_SQL)
    # run_id bound via CAST(:run_id AS bigint).
    assert "CAST(:run_id AS bigint)" in _SNAPSHOT_SQL
    # freshness window bound via named param inside make_interval (no cast needed).
    assert ":freshness_days" in _SNAPSHOT_SQL


# ── Writer return / finalisation contract ─────────────────────────────────────


def test_writer_returns_counters_and_finalises_run() -> None:
    assert "snapshotted" in _WRITER_SRC
    assert "price_change_events" in _WRITER_SRC
    assert "mark_done" in _WRITER_SRC
    assert "mark_failed" in _WRITER_SRC


# ── Migration 079 ─────────────────────────────────────────────────────────────


def test_migration_079_exists() -> None:
    assert _MIGRATION_079.is_file(), f"missing migration: {_MIGRATION_079}"


def test_migration_079_creates_three_schema_objects_and_view() -> None:
    sql = _MIGRATION_079.read_text("utf-8")
    # 1. snapshots table (plain — explicitly NOT partitioned, noted in header).
    assert "CREATE TABLE IF NOT EXISTS listing_source_snapshots" in sql
    assert "PRIMARY KEY (listing_source_id, snapshot_date)" in sql
    assert "REFERENCES listing_sources(id) ON DELETE CASCADE" in sql
    assert "REFERENCES scrape_runs(id) ON DELETE SET NULL" in sql
    assert "CREATE INDEX IF NOT EXISTS idx_lss_snapshot_date" in sql
    assert "CREATE INDEX IF NOT EXISTS idx_lss_source_date" in sql
    # 2. events table + CHECK + UNIQUE.
    assert "CREATE TABLE IF NOT EXISTS listing_source_events" in sql
    assert "id                bigserial   PRIMARY KEY" in sql.replace("\t", " ") or (
        "id" in sql and "bigserial" in sql and "PRIMARY KEY" in sql
    )
    assert "'price_change', 'delisted', 'relisted', 'edited', 'first_seen'" in sql
    assert "UNIQUE (listing_source_id, change_time, event_type)" in sql
    # 3. offer_price_history backlink column + index.
    assert "ALTER TABLE offer_price_history" in sql
    assert "ADD COLUMN IF NOT EXISTS listing_source_id bigint" in sql
    assert "idx_oph_source_change_time" in sql
    # 4. view.
    assert "CREATE OR REPLACE VIEW v_listing_source_price_on_date" in sql
    assert "JOIN listing_sources ls ON ls.id = lss.listing_source_id" in sql
    # Transactional + idempotent + partitioning note.
    assert "BEGIN;" in sql and "COMMIT;" in sql
    assert "НЕТ" in sql and "артицион" in sql  # partitioning explicitly deferred


def test_migration_079_seeds_schedule_enabled_true() -> None:
    sql = _MIGRATION_079.read_text("utf-8")
    assert "'listing_source_snapshot'" in sql
    assert "'{}'::jsonb" in sql
    assert "ON CONFLICT (source) DO NOTHING" in sql
    # enabled=true (SAFE — pure internal DB). Assert the seeded VALUES row has true,
    # not false (unlike the dormant yandex/avito sweeps).
    assert "true,                                        -- SAFE" in sql
    assert "false" not in sql.split("INSERT INTO scrape_schedules")[1]


def test_migration_079_uses_psycopg_safe_sql() -> None:
    """Migration is plain DDL (no bind params), but guard against accidental :x::type."""
    sql = _MIGRATION_079.read_text("utf-8")
    assert not re.search(r":\w+::", sql)


# ── Cheap behavioural test: counter logic via fake db ─────────────────────────


class _FakeResult:
    def __init__(self, rowcount: int) -> None:
        self.rowcount = rowcount


class _FakeDB:
    """Minimal stand-in for a SQLAlchemy Session — records execute() calls, returns rowcounts."""

    def __init__(self, rowcounts: list[int]) -> None:
        self._rowcounts = list(rowcounts)
        self.executed: list[Any] = []
        self.committed = False
        self.rolled_back = False

    def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
        self.executed.append((stmt, params))
        return _FakeResult(self._rowcounts.pop(0))

    def commit(self) -> None:
        self.committed = True

    def rollback(self) -> None:
        self.rolled_back = True


def test_counter_logic_with_fake_db(monkeypatch: pytest.MonkeyPatch) -> None:
    """snapshot_listing_sources maps the two execute() rowcounts to its counters + marks done."""
    marked: dict[str, Any] = {}
    monkeypatch.setattr(
        snap_mod.runs_mod,
        "mark_done",
        lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)),
    )
    monkeypatch.setattr(snap_mod.runs_mod, "mark_failed", lambda *a, **k: None)

    db = _FakeDB(rowcounts=[18355, 42])  # snapshot rowcount, then event rowcount
    out = snap_mod.snapshot_listing_sources(db, run_id=99)  # type: ignore[arg-type]

    assert out == {"snapshotted": 18355, "price_change_events": 42}
    assert db.committed is True
    assert len(db.executed) == 2
    # run_id threaded into the snapshot statement's bind params.
    _stmt, params = db.executed[0]
    assert params is not None and params["run_id"] == 99
    # Run finalised via mark_done with the same counters.
    assert marked["run_id"] == 99
    assert marked["counters"] == {"snapshotted": 18355, "price_change_events": 42}


def test_counter_logic_failure_path_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None:
    """On execute error: rollback + mark_failed + re-raise (no silent swallow)."""
    failed: dict[str, Any] = {}
    monkeypatch.setattr(snap_mod.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(
        snap_mod.runs_mod,
        "mark_failed",
        lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err),
    )

    class _BoomDB(_FakeDB):
        def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
            raise RuntimeError("boom")

    db = _BoomDB(rowcounts=[])
    with pytest.raises(RuntimeError, match="boom"):
        snap_mod.snapshot_listing_sources(db, run_id=7)  # type: ignore[arg-type]
    assert db.rolled_back is True
    assert failed["run_id"] == 7