from __future__ import annotations import inspect import os import re from pathlib import Path from typing import Any import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from app.tasks import deactivate_stale_avito as task_mod _SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql" _MIGRATION_115 = _SQL_DIR / "115_scrape_schedules_seed_deactivate_stale_yandex_cian.sql" _GENERIC_FN_SRC = inspect.getsource(task_mod.deactivate_stale_listings) _COMPAT_FN_SRC = inspect.getsource(task_mod.deactivate_stale_avito_listings) def test_segment_sql_uses_cast_interval_not_double_colon() -> None: sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text assert "CAST(:ttl_days || ' days' AS interval)" in sql assert not re.search(r"\w+::", sql) def test_segment_sql_uses_any_cast_text_array() -> None: sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text assert "ANY(CAST(:segments AS text[]))" in sql def test_segment_sql_filters_is_active_true() -> None: sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text assert "is_active = true" in sql def test_segment_sql_only_deactivates_not_deletes() -> None: sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text assert "SET is_active = false" in sql assert "DELETE" not in sql.upper() def test_all_segments_sql_uses_listing_source_param() -> None: sql = task_mod._DEACTIVATE_SQL_ALL_SEGMENTS.text assert ":listing_source" in sql assert "source = 'avito'" not in sql def test_deactivate_sql_alias_is_all_segments() -> None: assert task_mod._DEACTIVATE_SQL is task_mod._DEACTIVATE_SQL_ALL_SEGMENTS def test_generic_fn_exists() -> None: assert callable(task_mod.deactivate_stale_listings) def test_generic_fn_returns_deactivated_counter() -> None: assert "deactivated" in _GENERIC_FN_SRC def test_generic_fn_finalises_run() -> None: assert "mark_done" in _GENERIC_FN_SRC assert "mark_failed" in _GENERIC_FN_SRC def test_generic_fn_commits_and_rollbacks() -> None: assert "db.commit()" in _GENERIC_FN_SRC assert "db.rollback()" in _GENERIC_FN_SRC def test_compat_wrapper_calls_generic_with_avito_defaults() -> None: assert "deactivate_stale_listings(" in _COMPAT_FN_SRC assert 'listing_source="avito"' in _COMPAT_FN_SRC assert "settings.avito_stale_ttl_days" in _COMPAT_FN_SRC assert "segments=None" in _COMPAT_FN_SRC class _FakeResult: def __init__(self, rowcount: int) -> None: self.rowcount = rowcount class _FakeDB: def __init__(self, rowcount: int = 0) -> None: self._rowcount = rowcount self.executed: list[tuple[Any, Any]] = [] self.committed = False self.rolled_back = False def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult: self.executed.append((stmt, params)) return _FakeResult(self._rowcount) def commit(self) -> None: self.committed = True def rollback(self) -> None: self.rolled_back = True def test_vtorichka_stale_beyond_ttl_deactivated(monkeypatch: pytest.MonkeyPatch) -> None: marked: dict[str, Any] = {} monkeypatch.setattr( task_mod.runs_mod, "mark_done", lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)), ) monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None) db = _FakeDB(rowcount=5) out = task_mod.deactivate_stale_listings( db, run_id=1, listing_source="yandex", ttl_days=30, segments=["vtorichka"] ) # type: ignore[arg-type] assert out == {"deactivated": 5} assert db.committed is True _stmt, params = db.executed[0] assert params is not None assert params["segments"] == ["vtorichka"] assert params["listing_source"] == "yandex" assert params["ttl_days"] == 30 assert marked["counters"] == {"deactivated": 5} def test_novostroyki_not_in_segment_filter(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None) monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None) db = _FakeDB(rowcount=0) task_mod.deactivate_stale_listings( db, run_id=2, listing_source="yandex", ttl_days=30, segments=["vtorichka"] ) # type: ignore[arg-type] _stmt, params = db.executed[0] assert params is not None assert "segments" in params assert "novostroyki" not in params["segments"] def test_null_segment_not_in_filter(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None) monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None) db = _FakeDB(rowcount=0) task_mod.deactivate_stale_listings( db, run_id=3, listing_source="cian", ttl_days=30, segments=["vtorichka"] ) # type: ignore[arg-type] _stmt, params = db.executed[0] assert params is not None assert None not in params["segments"] def test_other_source_listing_source_param_matches(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None) monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None) db = _FakeDB(rowcount=0) task_mod.deactivate_stale_listings( db, run_id=4, listing_source="yandex", ttl_days=30, segments=["vtorichka"] ) # type: ignore[arg-type] _stmt, params = db.executed[0] assert params is not None assert params["listing_source"] == "yandex" def test_fresh_listing_zero_rowcount(monkeypatch: pytest.MonkeyPatch) -> None: marked: dict[str, Any] = {} monkeypatch.setattr( task_mod.runs_mod, "mark_done", lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)), ) monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None) db = _FakeDB(rowcount=0) out = task_mod.deactivate_stale_listings( db, run_id=5, listing_source="cian", ttl_days=30, segments=["vtorichka"] ) # type: ignore[arg-type] assert out == {"deactivated": 0} assert db.committed is True assert marked["counters"] == {"deactivated": 0} def test_segments_none_uses_all_segments_sql(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None) monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None) db = _FakeDB(rowcount=3) out = task_mod.deactivate_stale_listings( db, run_id=6, listing_source="avito", ttl_days=10, segments=None ) # type: ignore[arg-type] assert out == {"deactivated": 3} _stmt, params = db.executed[0] assert params is not None assert "segments" not in params assert params["listing_source"] == "avito" assert params["ttl_days"] == 10 def test_avito_compat_wrapper_counter(monkeypatch: pytest.MonkeyPatch) -> None: marked: dict[str, Any] = {} monkeypatch.setattr( task_mod.runs_mod, "mark_done", lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)), ) monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None) db = _FakeDB(rowcount=42) out = task_mod.deactivate_stale_avito_listings(db, run_id=7) # type: ignore[arg-type] assert out == {"deactivated": 42} assert db.committed is True _stmt, params = db.executed[0] assert params is not None assert params["listing_source"] == "avito" assert "segments" not in params assert marked["run_id"] == 7 def test_failure_path_rollback_and_mark_failed(monkeypatch: pytest.MonkeyPatch) -> None: failed: dict[str, Any] = {} monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None) monkeypatch.setattr( task_mod.runs_mod, "mark_failed", lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err), ) class _BoomDB(_FakeDB): def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult: raise RuntimeError("db exploded") db = _BoomDB() with pytest.raises(RuntimeError, match="db exploded"): task_mod.deactivate_stale_listings( db, run_id=8, listing_source="yandex", ttl_days=30, segments=["vtorichka"] ) # type: ignore[arg-type] assert db.rolled_back is True assert failed["run_id"] == 8 def test_migration_115_exists() -> None: assert _MIGRATION_115.is_file(), f"missing migration: {_MIGRATION_115}" def test_migration_115_seeds_yandex_and_cian() -> None: sql = _MIGRATION_115.read_text("utf-8") assert "'deactivate_stale_yandex'" in sql assert "'deactivate_stale_cian'" in sql def test_migration_115_is_idempotent() -> None: sql = _MIGRATION_115.read_text("utf-8") assert "ON CONFLICT (source) DO NOTHING" in sql def test_migration_115_enabled_true() -> None: sql = _MIGRATION_115.read_text("utf-8") assert "true" in sql def test_migration_115_is_transactional() -> None: sql = _MIGRATION_115.read_text("utf-8") assert "BEGIN;" in sql assert "COMMIT;" in sql def test_migration_115_window_7_to_8_utc() -> None: sql = _MIGRATION_115.read_text("utf-8") assert re.search(r"\b7\b", sql), "window_start_hour 7 missing" assert re.search(r"\b8\b", sql), "window_end_hour 8 missing" def test_migration_115_segments_vtorichka_in_default_params() -> None: sql = _MIGRATION_115.read_text("utf-8") assert "vtorichka" in sql def test_migration_115_ttl_30_days() -> None: sql = _MIGRATION_115.read_text("utf-8") assert "30" in sql def test_migration_115_no_psycopg_trap() -> None: sql = _MIGRATION_115.read_text("utf-8") assert not re.search(r":\w+::", sql) # ── staleness_column (#2204: domklik scraped_at vs last_seen_at) ─────────────── def _sql_text(stmt: Any) -> str: """Достать SQL-текст из TextClause, захваченного _FakeDB.execute.""" return str(getattr(stmt, "text", stmt)) def test_default_staleness_column_is_last_seen_at(monkeypatch: pytest.MonkeyPatch) -> None: """Без явного staleness_column WHERE фильтрует по last_seen_at (обратная совместимость).""" monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None) monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None) db = _FakeDB(rowcount=1) task_mod.deactivate_stale_listings( db, run_id=10, listing_source="avito", ttl_days=10, segments=None ) # type: ignore[arg-type] stmt, _params = db.executed[0] sql = _sql_text(stmt) assert "last_seen_at <" in sql assert "scraped_at" not in sql def test_scraped_at_column_used_in_all_segments_sql(monkeypatch: pytest.MonkeyPatch) -> None: """domklik bulk-touch: staleness_column='scraped_at' -> WHERE по scraped_at, не last_seen_at. Кейс: last_seen_at свежий (ежедневный bulk UPDATE), но scraped_at заморожен -> деактивация должна сработать по scraped_at. Проверяем, что именно эта колонка в SQL. """ marked: dict[str, Any] = {} monkeypatch.setattr( task_mod.runs_mod, "mark_done", lambda _db, run_id, counters: marked.update(counters=dict(counters)), ) monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None) db = _FakeDB(rowcount=6296) out = task_mod.deactivate_stale_listings( db, run_id=11, listing_source="domklik", ttl_days=14, segments=None, staleness_column="scraped_at", ) # type: ignore[arg-type] stmt, params = db.executed[0] sql = _sql_text(stmt) assert "scraped_at <" in sql assert "last_seen_at" not in sql assert params is not None assert params["listing_source"] == "domklik" assert params["ttl_days"] == 14 assert out == {"deactivated": 6296} assert marked["counters"] == {"deactivated": 6296} def test_scraped_at_column_used_in_segments_sql(monkeypatch: pytest.MonkeyPatch) -> None: """staleness_column='scraped_at' также уважается в segment-варианте SQL.""" monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None) monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None) db = _FakeDB(rowcount=0) task_mod.deactivate_stale_listings( db, run_id=12, listing_source="domklik", ttl_days=14, segments=["vtorichka"], staleness_column="scraped_at", ) # type: ignore[arg-type] stmt, _params = db.executed[0] sql = _sql_text(stmt) assert "scraped_at <" in sql assert "listing_segment = ANY(CAST(:segments AS text[]))" in sql def test_invalid_staleness_column_raises_and_marks_failed( monkeypatch: pytest.MonkeyPatch, ) -> None: """Невалидная колонка -> ValueError ДО SQL, run помечен failed, никакого execute.""" failed: dict[str, Any] = {} monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None) monkeypatch.setattr( task_mod.runs_mod, "mark_failed", lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err), ) db = _FakeDB(rowcount=0) with pytest.raises(ValueError, match="invalid staleness_column"): task_mod.deactivate_stale_listings( db, run_id=13, listing_source="domklik", ttl_days=14, segments=None, staleness_column="created_at; DROP TABLE listings", ) # type: ignore[arg-type] # Никакого SQL не выполнено (проверка ДО execute). assert db.executed == [] assert db.rolled_back is True assert failed["run_id"] == 13 def test_allowed_staleness_columns_whitelist() -> None: assert task_mod._ALLOWED_STALENESS_COLUMNS == frozenset({"last_seen_at", "scraped_at"}) # ── Migration 160 (deactivate_stale_domklik / _n1) ──────────────────────────── _MIGRATION_160 = _SQL_DIR / "160_seed_deactivate_stale_domklik_n1.sql" def test_migration_160_exists() -> None: assert _MIGRATION_160.is_file(), f"missing migration: {_MIGRATION_160}" def test_migration_160_seeds_domklik_and_n1() -> None: sql = _MIGRATION_160.read_text("utf-8") assert "'deactivate_stale_domklik'" in sql assert "'deactivate_stale_n1'" in sql def test_migration_160_domklik_uses_scraped_at() -> None: sql = _MIGRATION_160.read_text("utf-8") assert '"listing_source":"domklik"' in sql assert '"staleness_column":"scraped_at"' in sql def test_migration_160_n1_default_column_ttl14() -> None: sql = _MIGRATION_160.read_text("utf-8") assert '"listing_source":"n1"' in sql assert '"ttl_days":14' in sql def test_migration_160_is_idempotent() -> None: sql = _MIGRATION_160.read_text("utf-8") assert "ON CONFLICT (source) DO NOTHING" in sql def test_migration_160_is_transactional() -> None: sql = _MIGRATION_160.read_text("utf-8") assert "BEGIN;" in sql assert "COMMIT;" in sql def test_migration_160_no_psycopg_trap() -> None: sql = _MIGRATION_160.read_text("utf-8") assert not re.search(r":\w+::", sql)