gendesign/tradein-mvp/backend/tests/test_purge_expired_trade_in_data.py

"""ЭТАП 4 B2C launch — retention -> physical deletion task (part B).

Covers app/tasks/purge_expired_trade_in_data.py:
  - batched DELETE (not a single unbounded DELETE), each batch its own commit
  - stops draining a table once a batch returns fewer rows than batch_size
  - safety cap (max_batches) bounds a single run even on a huge backlog
  - both tables (trade_in_estimates, trade_in_leads) get drained
  - failure path: rollback + mark_failed with partial counters, exception re-raised
  - SQL shape: DELETE (not UPDATE/deactivate), no psycopg `::` cast trap

Style mirrors tests/test_deactivate_stale_listings.py (_FakeDB, monkeypatched
runs_mod.mark_done/mark_failed).
"""

from __future__ import annotations

import os
import re
from pathlib import Path
from typing import Any

import pytest

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from app.tasks import purge_expired_trade_in_data as task_mod

_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql"
_MIGRATION_193 = _SQL_DIR / "193_trade_in_privacy_retention.sql"


class _FakeResult:
    def __init__(self, rowcount: int) -> None:
        self.rowcount = rowcount


class _FakeDB:
    """Pops rowcounts in call order -- caller supplies the exact sequence expected."""

    def __init__(self, rowcounts: list[int]) -> None:
        self._rowcounts = list(rowcounts)
        self.executed: list[tuple[Any, Any]] = []
        self.commits = 0
        self.rolled_back = False

    def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
        self.executed.append((stmt, params))
        return _FakeResult(self._rowcounts.pop(0))

    def commit(self) -> None:
        self.commits += 1

    def rollback(self) -> None:
        self.rolled_back = True


def _patch_runs(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]:
    marked: dict[str, Any] = {}
    monkeypatch.setattr(
        task_mod.runs_mod,
        "mark_done",
        lambda _db, run_id, counters: marked.update(
            kind="done", run_id=run_id, counters=dict(counters)
        ),
    )
    monkeypatch.setattr(
        task_mod.runs_mod,
        "mark_failed",
        lambda _db, run_id, err, counters: marked.update(
            kind="failed", run_id=run_id, err=err, counters=dict(counters)
        ),
    )
    return marked


# ── batching behaviour ──────────────────────────────────────────────────────


def test_stops_when_batch_below_size(monkeypatch: pytest.MonkeyPatch) -> None:
    marked = _patch_runs(monkeypatch)
    # estimates: one batch of 3 (< batch_size=10) -> stop. leads: one batch of 0 -> stop.
    db = _FakeDB([3, 0])
    out = task_mod.purge_expired_trade_in_data(db, run_id=1, batch_size=10, max_batches=20)  # type: ignore[arg-type]
    assert out == {"estimates_deleted": 3, "leads_deleted": 0}
    assert len(db.executed) == 2
    assert db.commits == 2
    assert marked["counters"] == out


def test_loops_until_below_batch_size(monkeypatch: pytest.MonkeyPatch) -> None:
    _patch_runs(monkeypatch)
    # estimates: 5,5,2 (batch_size=5) -> 12 total, 3 batches. leads: 5,1 -> 6 total, 2 batches.
    db = _FakeDB([5, 5, 2, 5, 1])
    out = task_mod.purge_expired_trade_in_data(db, run_id=2, batch_size=5, max_batches=20)  # type: ignore[arg-type]
    assert out == {"estimates_deleted": 12, "leads_deleted": 6}
    assert len(db.executed) == 5
    assert db.commits == 5, "each batch must commit independently, not one final commit"


def test_respects_max_batches_cap(monkeypatch: pytest.MonkeyPatch) -> None:
    """Every batch returns a FULL batch_size (never 'caught up') -- only the
    max_batches safety cap can stop the loop. Proves the cap is enforced, not
    just coincidentally matching a 'caught up' condition."""
    _patch_runs(monkeypatch)
    db = _FakeDB([5, 5, 5, 5, 5, 5])  # exactly max_batches=3 per table, no more
    out = task_mod.purge_expired_trade_in_data(db, run_id=3, batch_size=5, max_batches=3)  # type: ignore[arg-type]
    assert out == {"estimates_deleted": 15, "leads_deleted": 15}
    assert len(db.executed) == 6  # 3 (estimates) + 3 (leads), NOT unbounded


def test_default_batch_size_and_max_batches_from_settings(monkeypatch: pytest.MonkeyPatch) -> None:
    _patch_runs(monkeypatch)
    db = _FakeDB([0, 0])  # first batch already empty on both tables -> stop immediately
    task_mod.purge_expired_trade_in_data(db, run_id=4)  # type: ignore[arg-type]
    _stmt, params = db.executed[0]
    assert params is not None
    assert params["batch_size"] == task_mod.settings.trade_in_purge_batch_size


# ── table coverage / SQL shape ──────────────────────────────────────────────


def test_drains_both_tables_in_order(monkeypatch: pytest.MonkeyPatch) -> None:
    _patch_runs(monkeypatch)
    db = _FakeDB([0, 0])
    task_mod.purge_expired_trade_in_data(db, run_id=5, batch_size=100, max_batches=1)  # type: ignore[arg-type]
    first_sql = str(getattr(db.executed[0][0], "text", db.executed[0][0]))
    second_sql = str(getattr(db.executed[1][0], "text", db.executed[1][0]))
    assert "trade_in_estimates" in first_sql
    assert "trade_in_leads" in second_sql


def test_estimates_sql_is_delete_not_update() -> None:
    sql = task_mod._DELETE_EXPIRED_ESTIMATES_SQL.text
    assert "DELETE FROM trade_in_estimates" in sql
    assert "UPDATE" not in sql.upper()
    assert "expires_at < NOW()" in sql
    assert "ORDER BY expires_at" in sql
    assert "LIMIT CAST(:batch_size AS int)" in sql
    assert not re.search(r":\w+::", sql)


def test_leads_sql_is_delete_not_update() -> None:
    sql = task_mod._DELETE_EXPIRED_LEADS_SQL.text
    assert "DELETE FROM trade_in_leads" in sql
    assert "UPDATE" not in sql.upper()
    assert "expires_at < NOW()" in sql
    assert "ORDER BY expires_at" in sql
    assert not re.search(r":\w+::", sql)


def test_sql_does_not_delete_whole_table_unbounded() -> None:
    """Neither statement is a bare `DELETE FROM table` -- both scope via a
    subselect + LIMIT batch."""
    statements = (
        task_mod._DELETE_EXPIRED_ESTIMATES_SQL.text,
        task_mod._DELETE_EXPIRED_LEADS_SQL.text,
    )
    for sql in statements:
        assert "WHERE id IN (" in sql
        assert "LIMIT" in sql


# ── failure path ─────────────────────────────────────────────────────────────


def test_failure_path_rollback_and_mark_failed(monkeypatch: pytest.MonkeyPatch) -> None:
    marked = _patch_runs(monkeypatch)

    class _BoomDB(_FakeDB):
        def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
            # First batch (estimates) succeeds and commits; second call (still
            # draining estimates, or first leads call) explodes.
            if len(self.executed) >= 1:
                raise RuntimeError("db exploded")
            return super().execute(stmt, params)

    db = _BoomDB([5])  # only ONE successful batch before the boom
    with pytest.raises(RuntimeError, match="db exploded"):
        task_mod.purge_expired_trade_in_data(db, run_id=6, batch_size=5, max_batches=20)  # type: ignore[arg-type]

    assert db.rolled_back is True
    assert marked["kind"] == "failed"
    assert marked["run_id"] == 6
    # Partial progress preserved in the counters passed to mark_failed (first
    # estimates batch of 5 already committed before the boom on the 2nd call).
    assert marked["counters"]["estimates_deleted"] == 5
    assert marked["counters"]["leads_deleted"] == 0


def test_idempotent_zero_rowcount_is_not_an_error(monkeypatch: pytest.MonkeyPatch) -> None:
    """Re-running against an already-drained backlog just deletes 0 rows, cleanly."""
    marked = _patch_runs(monkeypatch)
    db = _FakeDB([0, 0])
    out = task_mod.purge_expired_trade_in_data(db, run_id=7, batch_size=500, max_batches=20)  # type: ignore[arg-type]
    assert out == {"estimates_deleted": 0, "leads_deleted": 0}
    assert marked["kind"] == "done"


# ── migration 193 ────────────────────────────────────────────────────────────


def test_migration_193_exists() -> None:
    assert _MIGRATION_193.is_file(), f"missing migration: {_MIGRATION_193}"


def test_migration_193_is_transactional() -> None:
    sql = _MIGRATION_193.read_text("utf-8")
    assert "BEGIN;" in sql
    assert "COMMIT;" in sql


def test_migration_193_backfills_and_sets_not_null() -> None:
    sql = _MIGRATION_193.read_text("utf-8")
    assert "ADD COLUMN IF NOT EXISTS expires_at" in sql
    assert "WHERE expires_at IS NULL" in sql
    assert "SET NOT NULL" in sql
    assert "180 days" in sql


def test_migration_193_seeds_purge_schedule_disabled_by_default() -> None:
    sql = _MIGRATION_193.read_text("utf-8")
    assert "'purge_expired_trade_in_data'" in sql
    assert "ON CONFLICT (source) DO NOTHING" in sql
    # Seeded disabled -- first automated PII-DELETE job in trade-in deserves a
    # supervised first run before the scheduler can trigger it unattended.
    assert re.search(r"'purge_expired_trade_in_data',\s*\n\s*false,", sql)


def test_migration_193_no_psycopg_trap() -> None:
    sql = _MIGRATION_193.read_text("utf-8")
    assert not re.search(r":\w+::", sql)