gendesign/tradein-mvp/backend/tests/test_deactivate_stale_listings.py

from __future__ import annotations

import inspect
import os
import re
from pathlib import Path
from typing import Any

import pytest

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from app.tasks import deactivate_stale_avito as task_mod

_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql"
_MIGRATION_115 = _SQL_DIR / "115_scrape_schedules_seed_deactivate_stale_yandex_cian.sql"

_GENERIC_FN_SRC = inspect.getsource(task_mod.deactivate_stale_listings)
_COMPAT_FN_SRC = inspect.getsource(task_mod.deactivate_stale_avito_listings)


def test_segment_sql_uses_cast_interval_not_double_colon() -> None:
    sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text
    assert "CAST(:ttl_days || ' days' AS interval)" in sql
    assert not re.search(r"\w+::", sql)


def test_segment_sql_uses_any_cast_text_array() -> None:
    sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text
    assert "ANY(CAST(:segments AS text[]))" in sql


def test_segment_sql_filters_is_active_true() -> None:
    sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text
    assert "is_active = true" in sql


def test_segment_sql_only_deactivates_not_deletes() -> None:
    sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text
    assert "SET is_active = false" in sql
    assert "DELETE" not in sql.upper()


def test_all_segments_sql_uses_listing_source_param() -> None:
    sql = task_mod._DEACTIVATE_SQL_ALL_SEGMENTS.text
    assert ":listing_source" in sql
    assert "source = 'avito'" not in sql


def test_deactivate_sql_alias_is_all_segments() -> None:
    assert task_mod._DEACTIVATE_SQL is task_mod._DEACTIVATE_SQL_ALL_SEGMENTS


def test_generic_fn_exists() -> None:
    assert callable(task_mod.deactivate_stale_listings)


def test_generic_fn_returns_deactivated_counter() -> None:
    assert "deactivated" in _GENERIC_FN_SRC


def test_generic_fn_finalises_run() -> None:
    assert "mark_done" in _GENERIC_FN_SRC
    assert "mark_failed" in _GENERIC_FN_SRC


def test_generic_fn_commits_and_rollbacks() -> None:
    assert "db.commit()" in _GENERIC_FN_SRC
    assert "db.rollback()" in _GENERIC_FN_SRC


def test_compat_wrapper_calls_generic_with_avito_defaults() -> None:
    assert "deactivate_stale_listings(" in _COMPAT_FN_SRC
    assert 'listing_source="avito"' in _COMPAT_FN_SRC
    assert "settings.avito_stale_ttl_days" in _COMPAT_FN_SRC
    assert "segments=None" in _COMPAT_FN_SRC


class _FakeResult:
    def __init__(self, rowcount: int) -> None:
        self.rowcount = rowcount


class _FakeDB:
    def __init__(self, rowcount: int = 0) -> None:
        self._rowcount = rowcount
        self.executed: list[tuple[Any, Any]] = []
        self.committed = False
        self.rolled_back = False

    def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
        self.executed.append((stmt, params))
        return _FakeResult(self._rowcount)

    def commit(self) -> None:
        self.committed = True

    def rollback(self) -> None:
        self.rolled_back = True


def test_vtorichka_stale_beyond_ttl_deactivated(monkeypatch: pytest.MonkeyPatch) -> None:
    marked: dict[str, Any] = {}
    monkeypatch.setattr(
        task_mod.runs_mod,
        "mark_done",
        lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)),
    )
    monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
    db = _FakeDB(rowcount=5)
    out = task_mod.deactivate_stale_listings(
        db, run_id=1, listing_source="yandex", ttl_days=30, segments=["vtorichka"]
    )  # type: ignore[arg-type]
    assert out == {"deactivated": 5}
    assert db.committed is True
    _stmt, params = db.executed[0]
    assert params is not None
    assert params["segments"] == ["vtorichka"]
    assert params["listing_source"] == "yandex"
    assert params["ttl_days"] == 30
    assert marked["counters"] == {"deactivated": 5}


def test_novostroyki_not_in_segment_filter(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
    db = _FakeDB(rowcount=0)
    task_mod.deactivate_stale_listings(
        db, run_id=2, listing_source="yandex", ttl_days=30, segments=["vtorichka"]
    )  # type: ignore[arg-type]
    _stmt, params = db.executed[0]
    assert params is not None
    assert "segments" in params
    assert "novostroyki" not in params["segments"]


def test_null_segment_not_in_filter(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
    db = _FakeDB(rowcount=0)
    task_mod.deactivate_stale_listings(
        db, run_id=3, listing_source="cian", ttl_days=30, segments=["vtorichka"]
    )  # type: ignore[arg-type]
    _stmt, params = db.executed[0]
    assert params is not None
    assert None not in params["segments"]


def test_other_source_listing_source_param_matches(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
    db = _FakeDB(rowcount=0)
    task_mod.deactivate_stale_listings(
        db, run_id=4, listing_source="yandex", ttl_days=30, segments=["vtorichka"]
    )  # type: ignore[arg-type]
    _stmt, params = db.executed[0]
    assert params is not None
    assert params["listing_source"] == "yandex"


def test_fresh_listing_zero_rowcount(monkeypatch: pytest.MonkeyPatch) -> None:
    marked: dict[str, Any] = {}
    monkeypatch.setattr(
        task_mod.runs_mod,
        "mark_done",
        lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)),
    )
    monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
    db = _FakeDB(rowcount=0)
    out = task_mod.deactivate_stale_listings(
        db, run_id=5, listing_source="cian", ttl_days=30, segments=["vtorichka"]
    )  # type: ignore[arg-type]
    assert out == {"deactivated": 0}
    assert db.committed is True
    assert marked["counters"] == {"deactivated": 0}


def test_segments_none_uses_all_segments_sql(monkeypatch: pytest.MonkeyPatch) -> None:
    monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
    db = _FakeDB(rowcount=3)
    out = task_mod.deactivate_stale_listings(
        db, run_id=6, listing_source="avito", ttl_days=10, segments=None
    )  # type: ignore[arg-type]
    assert out == {"deactivated": 3}
    _stmt, params = db.executed[0]
    assert params is not None
    assert "segments" not in params
    assert params["listing_source"] == "avito"
    assert params["ttl_days"] == 10


def test_avito_compat_wrapper_counter(monkeypatch: pytest.MonkeyPatch) -> None:
    marked: dict[str, Any] = {}
    monkeypatch.setattr(
        task_mod.runs_mod,
        "mark_done",
        lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)),
    )
    monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
    db = _FakeDB(rowcount=42)
    out = task_mod.deactivate_stale_avito_listings(db, run_id=7)  # type: ignore[arg-type]
    assert out == {"deactivated": 42}
    assert db.committed is True
    _stmt, params = db.executed[0]
    assert params is not None
    assert params["listing_source"] == "avito"
    assert "segments" not in params
    assert marked["run_id"] == 7


def test_failure_path_rollback_and_mark_failed(monkeypatch: pytest.MonkeyPatch) -> None:
    failed: dict[str, Any] = {}
    monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(
        task_mod.runs_mod,
        "mark_failed",
        lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err),
    )

    class _BoomDB(_FakeDB):
        def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
            raise RuntimeError("db exploded")

    db = _BoomDB()
    with pytest.raises(RuntimeError, match="db exploded"):
        task_mod.deactivate_stale_listings(
            db, run_id=8, listing_source="yandex", ttl_days=30, segments=["vtorichka"]
        )  # type: ignore[arg-type]
    assert db.rolled_back is True
    assert failed["run_id"] == 8


def test_migration_115_exists() -> None:
    assert _MIGRATION_115.is_file(), f"missing migration: {_MIGRATION_115}"


def test_migration_115_seeds_yandex_and_cian() -> None:
    sql = _MIGRATION_115.read_text("utf-8")
    assert "'deactivate_stale_yandex'" in sql
    assert "'deactivate_stale_cian'" in sql


def test_migration_115_is_idempotent() -> None:
    sql = _MIGRATION_115.read_text("utf-8")
    assert "ON CONFLICT (source) DO NOTHING" in sql


def test_migration_115_enabled_true() -> None:
    sql = _MIGRATION_115.read_text("utf-8")
    assert "true" in sql


def test_migration_115_is_transactional() -> None:
    sql = _MIGRATION_115.read_text("utf-8")
    assert "BEGIN;" in sql
    assert "COMMIT;" in sql


def test_migration_115_window_7_to_8_utc() -> None:
    sql = _MIGRATION_115.read_text("utf-8")
    assert re.search(r"\b7\b", sql), "window_start_hour 7 missing"
    assert re.search(r"\b8\b", sql), "window_end_hour 8 missing"


def test_migration_115_segments_vtorichka_in_default_params() -> None:
    sql = _MIGRATION_115.read_text("utf-8")
    assert "vtorichka" in sql


def test_migration_115_ttl_30_days() -> None:
    sql = _MIGRATION_115.read_text("utf-8")
    assert "30" in sql


def test_migration_115_no_psycopg_trap() -> None:
    sql = _MIGRATION_115.read_text("utf-8")
    assert not re.search(r":\w+::", sql)


# ── staleness_column (#2204: domklik scraped_at vs last_seen_at) ───────────────


def _sql_text(stmt: Any) -> str:
    """Достать SQL-текст из TextClause, захваченного _FakeDB.execute."""
    return str(getattr(stmt, "text", stmt))


def test_default_staleness_column_is_last_seen_at(monkeypatch: pytest.MonkeyPatch) -> None:
    """Без явного staleness_column WHERE фильтрует по last_seen_at (обратная совместимость)."""
    monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
    db = _FakeDB(rowcount=1)
    task_mod.deactivate_stale_listings(
        db, run_id=10, listing_source="avito", ttl_days=10, segments=None
    )  # type: ignore[arg-type]
    stmt, _params = db.executed[0]
    sql = _sql_text(stmt)
    assert "last_seen_at <" in sql
    assert "scraped_at" not in sql


def test_scraped_at_column_used_in_all_segments_sql(monkeypatch: pytest.MonkeyPatch) -> None:
    """domklik bulk-touch: staleness_column='scraped_at' -> WHERE по scraped_at, не last_seen_at.

    Кейс: last_seen_at свежий (ежедневный bulk UPDATE), но scraped_at заморожен ->
    деактивация должна сработать по scraped_at. Проверяем, что именно эта колонка в SQL.
    """
    marked: dict[str, Any] = {}
    monkeypatch.setattr(
        task_mod.runs_mod,
        "mark_done",
        lambda _db, run_id, counters: marked.update(counters=dict(counters)),
    )
    monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
    db = _FakeDB(rowcount=6296)
    out = task_mod.deactivate_stale_listings(
        db,
        run_id=11,
        listing_source="domklik",
        ttl_days=14,
        segments=None,
        staleness_column="scraped_at",
    )  # type: ignore[arg-type]
    stmt, params = db.executed[0]
    sql = _sql_text(stmt)
    assert "scraped_at <" in sql
    assert "last_seen_at" not in sql
    assert params is not None
    assert params["listing_source"] == "domklik"
    assert params["ttl_days"] == 14
    assert out == {"deactivated": 6296}
    assert marked["counters"] == {"deactivated": 6296}


def test_scraped_at_column_used_in_segments_sql(monkeypatch: pytest.MonkeyPatch) -> None:
    """staleness_column='scraped_at' также уважается в segment-варианте SQL."""
    monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
    db = _FakeDB(rowcount=0)
    task_mod.deactivate_stale_listings(
        db,
        run_id=12,
        listing_source="domklik",
        ttl_days=14,
        segments=["vtorichka"],
        staleness_column="scraped_at",
    )  # type: ignore[arg-type]
    stmt, _params = db.executed[0]
    sql = _sql_text(stmt)
    assert "scraped_at <" in sql
    assert "listing_segment = ANY(CAST(:segments AS text[]))" in sql


def test_invalid_staleness_column_raises_and_marks_failed(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Невалидная колонка -> ValueError ДО SQL, run помечен failed, никакого execute."""
    failed: dict[str, Any] = {}
    monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
    monkeypatch.setattr(
        task_mod.runs_mod,
        "mark_failed",
        lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err),
    )
    db = _FakeDB(rowcount=0)
    with pytest.raises(ValueError, match="invalid staleness_column"):
        task_mod.deactivate_stale_listings(
            db,
            run_id=13,
            listing_source="domklik",
            ttl_days=14,
            segments=None,
            staleness_column="created_at; DROP TABLE listings",
        )  # type: ignore[arg-type]
    # Никакого SQL не выполнено (проверка ДО execute).
    assert db.executed == []
    assert db.rolled_back is True
    assert failed["run_id"] == 13


def test_allowed_staleness_columns_whitelist() -> None:
    assert task_mod._ALLOWED_STALENESS_COLUMNS == frozenset({"last_seen_at", "scraped_at"})


# ── Migration 160 (deactivate_stale_domklik / _n1) ────────────────────────────

_MIGRATION_160 = _SQL_DIR / "160_seed_deactivate_stale_domklik_n1.sql"


def test_migration_160_exists() -> None:
    assert _MIGRATION_160.is_file(), f"missing migration: {_MIGRATION_160}"


def test_migration_160_seeds_domklik_and_n1() -> None:
    sql = _MIGRATION_160.read_text("utf-8")
    assert "'deactivate_stale_domklik'" in sql
    assert "'deactivate_stale_n1'" in sql


def test_migration_160_domklik_uses_scraped_at() -> None:
    sql = _MIGRATION_160.read_text("utf-8")
    assert '"listing_source":"domklik"' in sql
    assert '"staleness_column":"scraped_at"' in sql


def test_migration_160_n1_default_column_ttl14() -> None:
    sql = _MIGRATION_160.read_text("utf-8")
    assert '"listing_source":"n1"' in sql
    assert '"ttl_days":14' in sql


def test_migration_160_is_idempotent() -> None:
    sql = _MIGRATION_160.read_text("utf-8")
    assert "ON CONFLICT (source) DO NOTHING" in sql


def test_migration_160_is_transactional() -> None:
    sql = _MIGRATION_160.read_text("utf-8")
    assert "BEGIN;" in sql
    assert "COMMIT;" in sql


def test_migration_160_no_psycopg_trap() -> None:
    sql = _MIGRATION_160.read_text("utf-8")
    assert not re.search(r":\w+::", sql)