281 lines
12 KiB
Python
281 lines
12 KiB
Python
"""Tests for the per-source listing price-history snapshot writer (#570).
|
||
|
||
Writer (snapshot_listing_sources) is SQL-heavy, so most assertions are static: we read
|
||
the emitted SQL via inspect/text-attr and check the upsert + event-diff CTE shape, table/
|
||
column names, and the psycopg-v3 cast discipline (no :param::type). We also assert the
|
||
scheduler wiring (trigger fn + dispatch branch) and the migration 079 contents.
|
||
|
||
Plus one cheap behavioural test: a fake db (monkeypatched .execute) drives the counter
|
||
logic without a real Postgres.
|
||
|
||
Static style mirrors tests/test_rosreestr_dedup_key.py / test_yandex_city_sweep.py.
|
||
"""
|
||
|
||
import inspect
|
||
import os
|
||
import re
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
import pytest
|
||
|
||
# Importing app.services.scheduler / app.tasks pulls app.core.config.Settings → needs
|
||
# DATABASE_URL. Stub it BEFORE app imports (as in test_scheduler.py) — these tests are
|
||
# static / fake-db; no live database is touched.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||
|
||
from app.services import scheduler
|
||
from app.tasks import listing_source_snapshot as snap_mod
|
||
|
||
_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql"
|
||
_MIGRATION_079 = _SQL_DIR / "079_listing_source_history.sql"
|
||
|
||
# Emitted SQL text (SQLAlchemy text() clause → .text gives the raw string).
|
||
_SNAPSHOT_SQL = str(snap_mod._SNAPSHOT_SQL.text)
|
||
_EVENT_DIFF_SQL = str(snap_mod._EVENT_DIFF_SQL.text)
|
||
_ALL_SQL = _SNAPSHOT_SQL + "\n" + _EVENT_DIFF_SQL
|
||
_WRITER_SRC = inspect.getsource(snap_mod.snapshot_listing_sources)
|
||
|
||
|
||
# ── Snapshot upsert SQL ───────────────────────────────────────────────────────
|
||
|
||
|
||
def test_snapshot_targets_correct_table_and_columns() -> None:
|
||
assert "INSERT INTO listing_source_snapshots" in _SNAPSHOT_SQL
|
||
for col in (
|
||
"listing_source_id",
|
||
"snapshot_date",
|
||
"price_rub",
|
||
"is_active",
|
||
"last_seen_at",
|
||
"payload_hash",
|
||
"observed_at",
|
||
"run_id",
|
||
):
|
||
assert col in _SNAPSHOT_SQL, f"snapshot SQL missing column {col!r}"
|
||
assert "FROM listing_sources" in _SNAPSHOT_SQL
|
||
assert "CURRENT_DATE" in _SNAPSHOT_SQL
|
||
|
||
|
||
def test_snapshot_is_upsert_last_write_wins() -> None:
|
||
"""ON CONFLICT (listing_source_id, snapshot_date) DO UPDATE — last-write-wins for the day."""
|
||
assert "ON CONFLICT (listing_source_id, snapshot_date) DO UPDATE" in _SNAPSHOT_SQL
|
||
# Collapse runs of whitespace so column-alignment spacing doesn't matter.
|
||
flat = re.sub(r"\s+", " ", _SNAPSHOT_SQL)
|
||
for col in ("price_rub", "is_active", "last_seen_at", "payload_hash", "observed_at", "run_id"):
|
||
assert f"{col} = EXCLUDED.{col}" in flat, f"upsert SET missing {col}"
|
||
|
||
|
||
def test_snapshot_derives_is_active_and_payload_hash() -> None:
|
||
# is_active derived from freshness window; payload_hash = md5(raw_payload::text).
|
||
assert "make_interval(days => :freshness_days)" in _SNAPSHOT_SQL
|
||
assert "AS is_active" in _SNAPSHOT_SQL
|
||
assert "md5(raw_payload::text)" in _SNAPSHOT_SQL
|
||
|
||
|
||
# ── Event-diff CTE SQL ────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_event_diff_is_set_based_cte_not_python_loop() -> None:
|
||
"""Event diff is one set-based INSERT … SELECT over a CTE — never a row-by-row loop."""
|
||
assert "WITH today AS" in _EVENT_DIFF_SQL
|
||
assert "prior AS" in _EVENT_DIFF_SQL
|
||
assert "DISTINCT ON (listing_source_id)" in _EVENT_DIFF_SQL
|
||
assert "INSERT INTO listing_source_events" in _EVENT_DIFF_SQL
|
||
# Prior = most-recent snapshot strictly before today.
|
||
assert "snapshot_date < CURRENT_DATE" in _EVENT_DIFF_SQL
|
||
assert "snapshot_date = CURRENT_DATE" in _EVENT_DIFF_SQL
|
||
assert "ORDER BY listing_source_id, snapshot_date DESC" in _EVENT_DIFF_SQL
|
||
# No Python iteration over rows in the writer body (set-based only).
|
||
body = _WRITER_SRC.split('"""', 2)[-1]
|
||
assert "for " not in body, "writer must be set-based — no Python row loop"
|
||
|
||
|
||
def test_event_diff_emits_price_change_with_diff_percent() -> None:
|
||
assert "'price_change'" in _EVENT_DIFF_SQL
|
||
# diff_percent = (new-old)/old*100.
|
||
assert "(t.price_rub - p.price_rub)" in _EVENT_DIFF_SQL
|
||
assert "/ p.price_rub * 100" in _EVENT_DIFF_SQL
|
||
# Only when the price actually changed and old is a usable denominator.
|
||
assert "t.price_rub <> p.price_rub" in _EVENT_DIFF_SQL
|
||
assert "p.price_rub <> 0" in _EVENT_DIFF_SQL
|
||
assert "p.price_rub IS NOT NULL" in _EVENT_DIFF_SQL
|
||
|
||
|
||
def test_event_diff_is_idempotent_on_conflict() -> None:
|
||
assert "ON CONFLICT (listing_source_id, change_time, event_type) DO NOTHING" in _EVENT_DIFF_SQL
|
||
|
||
|
||
# ── psycopg v3 cast discipline ────────────────────────────────────────────────
|
||
|
||
|
||
def test_sql_uses_psycopg_v3_casts_not_double_colon() -> None:
|
||
"""psycopg v3: bind params via CAST(:x AS type), never :x::type.
|
||
|
||
Literal ::-casts on columns (raw_payload::text, ::numeric) are fine — the ban is
|
||
only on :param::type, which psycopg v3 breaks.
|
||
"""
|
||
# No bind-param (:name) followed by ::type anywhere in the emitted SQL.
|
||
assert not re.search(r":\w+::", _ALL_SQL)
|
||
# run_id bound via CAST(:run_id AS bigint).
|
||
assert "CAST(:run_id AS bigint)" in _SNAPSHOT_SQL
|
||
# freshness window bound via named param inside make_interval (no cast needed).
|
||
assert ":freshness_days" in _SNAPSHOT_SQL
|
||
|
||
|
||
# ── Writer return / finalisation contract ─────────────────────────────────────
|
||
|
||
|
||
def test_writer_returns_counters_and_finalises_run() -> None:
|
||
assert "snapshotted" in _WRITER_SRC
|
||
assert "price_change_events" in _WRITER_SRC
|
||
assert "mark_done" in _WRITER_SRC
|
||
assert "mark_failed" in _WRITER_SRC
|
||
|
||
|
||
# ── Scheduler wiring ──────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_trigger_fn_exists_and_is_async() -> None:
|
||
fn = getattr(scheduler, "trigger_listing_source_snapshot_run", None)
|
||
assert fn is not None, "trigger_listing_source_snapshot_run missing from scheduler"
|
||
assert inspect.iscoroutinefunction(fn)
|
||
|
||
|
||
def test_dispatch_branch_wired() -> None:
|
||
"""scheduler_loop dispatches source='listing_source_snapshot' to the trigger."""
|
||
loop_src = inspect.getsource(scheduler.scheduler_loop)
|
||
assert 'source == "listing_source_snapshot"' in loop_src
|
||
assert "trigger_listing_source_snapshot_run(db, sch)" in loop_src
|
||
|
||
|
||
def test_trigger_runs_sync_task_in_executor() -> None:
|
||
"""Sync DB-only task → run_in_executor (mirror rosreestr), not a bare await."""
|
||
trig_src = inspect.getsource(scheduler.trigger_listing_source_snapshot_run)
|
||
assert "run_in_executor(None, snapshot_listing_sources" in trig_src
|
||
assert "_claim_run(db, schedule_row)" in trig_src
|
||
|
||
|
||
# ── Migration 079 ─────────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_migration_079_exists() -> None:
|
||
assert _MIGRATION_079.is_file(), f"missing migration: {_MIGRATION_079}"
|
||
|
||
|
||
def test_migration_079_creates_three_schema_objects_and_view() -> None:
|
||
sql = _MIGRATION_079.read_text("utf-8")
|
||
# 1. snapshots table (plain — explicitly NOT partitioned, noted in header).
|
||
assert "CREATE TABLE IF NOT EXISTS listing_source_snapshots" in sql
|
||
assert "PRIMARY KEY (listing_source_id, snapshot_date)" in sql
|
||
assert "REFERENCES listing_sources(id) ON DELETE CASCADE" in sql
|
||
assert "REFERENCES scrape_runs(id) ON DELETE SET NULL" in sql
|
||
assert "CREATE INDEX IF NOT EXISTS idx_lss_snapshot_date" in sql
|
||
assert "CREATE INDEX IF NOT EXISTS idx_lss_source_date" in sql
|
||
# 2. events table + CHECK + UNIQUE.
|
||
assert "CREATE TABLE IF NOT EXISTS listing_source_events" in sql
|
||
assert "id bigserial PRIMARY KEY" in sql.replace("\t", " ") or (
|
||
"id" in sql and "bigserial" in sql and "PRIMARY KEY" in sql
|
||
)
|
||
assert "'price_change', 'delisted', 'relisted', 'edited', 'first_seen'" in sql
|
||
assert "UNIQUE (listing_source_id, change_time, event_type)" in sql
|
||
# 3. offer_price_history backlink column + index.
|
||
assert "ALTER TABLE offer_price_history" in sql
|
||
assert "ADD COLUMN IF NOT EXISTS listing_source_id bigint" in sql
|
||
assert "idx_oph_source_change_time" in sql
|
||
# 4. view.
|
||
assert "CREATE OR REPLACE VIEW v_listing_source_price_on_date" in sql
|
||
assert "JOIN listing_sources ls ON ls.id = lss.listing_source_id" in sql
|
||
# Transactional + idempotent + partitioning note.
|
||
assert "BEGIN;" in sql and "COMMIT;" in sql
|
||
assert "НЕТ" in sql and "артицион" in sql # partitioning explicitly deferred
|
||
|
||
|
||
def test_migration_079_seeds_schedule_enabled_true() -> None:
|
||
sql = _MIGRATION_079.read_text("utf-8")
|
||
assert "'listing_source_snapshot'" in sql
|
||
assert "'{}'::jsonb" in sql
|
||
assert "ON CONFLICT (source) DO NOTHING" in sql
|
||
# enabled=true (SAFE — pure internal DB). Assert the seeded VALUES row has true,
|
||
# not false (unlike the dormant yandex/avito sweeps).
|
||
assert "true, -- SAFE" in sql
|
||
assert "false" not in sql.split("INSERT INTO scrape_schedules")[1]
|
||
|
||
|
||
def test_migration_079_uses_psycopg_safe_sql() -> None:
|
||
"""Migration is plain DDL (no bind params), but guard against accidental :x::type."""
|
||
sql = _MIGRATION_079.read_text("utf-8")
|
||
assert not re.search(r":\w+::", sql)
|
||
|
||
|
||
# ── Cheap behavioural test: counter logic via fake db ─────────────────────────
|
||
|
||
|
||
class _FakeResult:
|
||
def __init__(self, rowcount: int) -> None:
|
||
self.rowcount = rowcount
|
||
|
||
|
||
class _FakeDB:
|
||
"""Minimal stand-in for a SQLAlchemy Session — records execute() calls, returns rowcounts."""
|
||
|
||
def __init__(self, rowcounts: list[int]) -> None:
|
||
self._rowcounts = list(rowcounts)
|
||
self.executed: list[Any] = []
|
||
self.committed = False
|
||
self.rolled_back = False
|
||
|
||
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
|
||
self.executed.append((stmt, params))
|
||
return _FakeResult(self._rowcounts.pop(0))
|
||
|
||
def commit(self) -> None:
|
||
self.committed = True
|
||
|
||
def rollback(self) -> None:
|
||
self.rolled_back = True
|
||
|
||
|
||
def test_counter_logic_with_fake_db(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""snapshot_listing_sources maps the two execute() rowcounts to its counters + marks done."""
|
||
marked: dict[str, Any] = {}
|
||
monkeypatch.setattr(
|
||
snap_mod.runs_mod,
|
||
"mark_done",
|
||
lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)),
|
||
)
|
||
monkeypatch.setattr(snap_mod.runs_mod, "mark_failed", lambda *a, **k: None)
|
||
|
||
db = _FakeDB(rowcounts=[18355, 42]) # snapshot rowcount, then event rowcount
|
||
out = snap_mod.snapshot_listing_sources(db, run_id=99) # type: ignore[arg-type]
|
||
|
||
assert out == {"snapshotted": 18355, "price_change_events": 42}
|
||
assert db.committed is True
|
||
assert len(db.executed) == 2
|
||
# run_id threaded into the snapshot statement's bind params.
|
||
_stmt, params = db.executed[0]
|
||
assert params is not None and params["run_id"] == 99
|
||
# Run finalised via mark_done with the same counters.
|
||
assert marked["run_id"] == 99
|
||
assert marked["counters"] == {"snapshotted": 18355, "price_change_events": 42}
|
||
|
||
|
||
def test_counter_logic_failure_path_marks_failed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||
"""On execute error: rollback + mark_failed + re-raise (no silent swallow)."""
|
||
failed: dict[str, Any] = {}
|
||
monkeypatch.setattr(snap_mod.runs_mod, "mark_done", lambda *a, **k: None)
|
||
monkeypatch.setattr(
|
||
snap_mod.runs_mod,
|
||
"mark_failed",
|
||
lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err),
|
||
)
|
||
|
||
class _BoomDB(_FakeDB):
|
||
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
|
||
raise RuntimeError("boom")
|
||
|
||
db = _BoomDB(rowcounts=[])
|
||
with pytest.raises(RuntimeError, match="boom"):
|
||
snap_mod.snapshot_listing_sources(db, run_id=7) # type: ignore[arg-type]
|
||
assert db.rolled_back is True
|
||
assert failed["run_id"] == 7
|