lekss361
91f9a95f93
All checks were successful
Deploy Trade-In / changes (push) Successful in 7s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / deploy (push) Successful in 45s
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 33s
Deploy Trade-In / build-backend (push) Successful in 56s
305 lines
11 KiB
Python
305 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import inspect
|
|
import os
|
|
import re
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
|
|
|
from app.services import scheduler
|
|
from app.tasks import deactivate_stale_avito as task_mod
|
|
|
|
_SQL_DIR = Path(__file__).resolve().parents[1] / "data" / "sql"
|
|
_MIGRATION_115 = _SQL_DIR / "115_scrape_schedules_seed_deactivate_stale_yandex_cian.sql"
|
|
|
|
_GENERIC_FN_SRC = inspect.getsource(task_mod.deactivate_stale_listings)
|
|
_COMPAT_FN_SRC = inspect.getsource(task_mod.deactivate_stale_avito_listings)
|
|
|
|
|
|
def test_segment_sql_uses_cast_interval_not_double_colon() -> None:
|
|
sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text
|
|
assert "CAST(:ttl_days || ' days' AS interval)" in sql
|
|
assert not re.search(r"\w+::", sql)
|
|
|
|
|
|
def test_segment_sql_uses_any_cast_text_array() -> None:
|
|
sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text
|
|
assert "ANY(CAST(:segments AS text[]))" in sql
|
|
|
|
|
|
def test_segment_sql_filters_is_active_true() -> None:
|
|
sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text
|
|
assert "is_active = true" in sql
|
|
|
|
|
|
def test_segment_sql_only_deactivates_not_deletes() -> None:
|
|
sql = task_mod._DEACTIVATE_SQL_SEGMENTS.text
|
|
assert "SET is_active = false" in sql
|
|
assert "DELETE" not in sql.upper()
|
|
|
|
|
|
def test_all_segments_sql_uses_listing_source_param() -> None:
|
|
sql = task_mod._DEACTIVATE_SQL_ALL_SEGMENTS.text
|
|
assert ":listing_source" in sql
|
|
assert "source = 'avito'" not in sql
|
|
|
|
|
|
def test_deactivate_sql_alias_is_all_segments() -> None:
|
|
assert task_mod._DEACTIVATE_SQL is task_mod._DEACTIVATE_SQL_ALL_SEGMENTS
|
|
|
|
|
|
def test_generic_fn_exists() -> None:
|
|
assert callable(task_mod.deactivate_stale_listings)
|
|
|
|
|
|
def test_generic_fn_returns_deactivated_counter() -> None:
|
|
assert "deactivated" in _GENERIC_FN_SRC
|
|
|
|
|
|
def test_generic_fn_finalises_run() -> None:
|
|
assert "mark_done" in _GENERIC_FN_SRC
|
|
assert "mark_failed" in _GENERIC_FN_SRC
|
|
|
|
|
|
def test_generic_fn_commits_and_rollbacks() -> None:
|
|
assert "db.commit()" in _GENERIC_FN_SRC
|
|
assert "db.rollback()" in _GENERIC_FN_SRC
|
|
|
|
|
|
def test_compat_wrapper_calls_generic_with_avito_defaults() -> None:
|
|
assert "deactivate_stale_listings(" in _COMPAT_FN_SRC
|
|
assert 'listing_source="avito"' in _COMPAT_FN_SRC
|
|
assert "settings.avito_stale_ttl_days" in _COMPAT_FN_SRC
|
|
assert "segments=None" in _COMPAT_FN_SRC
|
|
|
|
|
|
def test_trigger_deactivate_stale_run_exists_and_is_async() -> None:
|
|
fn = getattr(scheduler, "trigger_deactivate_stale_run", None)
|
|
assert fn is not None, "trigger_deactivate_stale_run missing from scheduler"
|
|
assert inspect.iscoroutinefunction(fn)
|
|
|
|
|
|
def test_dispatch_branch_uses_startswith() -> None:
|
|
loop_src = inspect.getsource(scheduler.scheduler_loop)
|
|
assert 'source.startswith("deactivate_stale_")' in loop_src
|
|
assert "trigger_deactivate_stale_run(db, sch)" in loop_src
|
|
|
|
|
|
def test_trigger_runs_sync_task_in_executor() -> None:
|
|
trig_src = inspect.getsource(scheduler.trigger_deactivate_stale_run)
|
|
assert "run_in_executor(" in trig_src
|
|
assert "deactivate_stale_listings" in trig_src
|
|
assert "_claim_run(db, schedule_row)" in trig_src
|
|
|
|
|
|
def test_trigger_reads_params_from_default_params() -> None:
|
|
trig_src = inspect.getsource(scheduler.trigger_deactivate_stale_run)
|
|
assert "listing_source" in trig_src
|
|
assert "ttl_days" in trig_src
|
|
assert "segments" in trig_src
|
|
|
|
|
|
class _FakeResult:
|
|
def __init__(self, rowcount: int) -> None:
|
|
self.rowcount = rowcount
|
|
|
|
|
|
class _FakeDB:
|
|
def __init__(self, rowcount: int = 0) -> None:
|
|
self._rowcount = rowcount
|
|
self.executed: list[tuple[Any, Any]] = []
|
|
self.committed = False
|
|
self.rolled_back = False
|
|
|
|
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
|
|
self.executed.append((stmt, params))
|
|
return _FakeResult(self._rowcount)
|
|
|
|
def commit(self) -> None:
|
|
self.committed = True
|
|
|
|
def rollback(self) -> None:
|
|
self.rolled_back = True
|
|
|
|
|
|
def test_vtorichka_stale_beyond_ttl_deactivated(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
marked: dict[str, Any] = {}
|
|
monkeypatch.setattr(
|
|
task_mod.runs_mod,
|
|
"mark_done",
|
|
lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)),
|
|
)
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
|
|
db = _FakeDB(rowcount=5)
|
|
out = task_mod.deactivate_stale_listings(
|
|
db, run_id=1, listing_source="yandex", ttl_days=30, segments=["vtorichka"]
|
|
) # type: ignore[arg-type]
|
|
assert out == {"deactivated": 5}
|
|
assert db.committed is True
|
|
_stmt, params = db.executed[0]
|
|
assert params is not None
|
|
assert params["segments"] == ["vtorichka"]
|
|
assert params["listing_source"] == "yandex"
|
|
assert params["ttl_days"] == 30
|
|
assert marked["counters"] == {"deactivated": 5}
|
|
|
|
|
|
def test_novostroyki_not_in_segment_filter(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
|
|
db = _FakeDB(rowcount=0)
|
|
task_mod.deactivate_stale_listings(
|
|
db, run_id=2, listing_source="yandex", ttl_days=30, segments=["vtorichka"]
|
|
) # type: ignore[arg-type]
|
|
_stmt, params = db.executed[0]
|
|
assert params is not None
|
|
assert "segments" in params
|
|
assert "novostroyki" not in params["segments"]
|
|
|
|
|
|
def test_null_segment_not_in_filter(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
|
|
db = _FakeDB(rowcount=0)
|
|
task_mod.deactivate_stale_listings(
|
|
db, run_id=3, listing_source="cian", ttl_days=30, segments=["vtorichka"]
|
|
) # type: ignore[arg-type]
|
|
_stmt, params = db.executed[0]
|
|
assert params is not None
|
|
assert None not in params["segments"]
|
|
|
|
|
|
def test_other_source_listing_source_param_matches(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
|
|
db = _FakeDB(rowcount=0)
|
|
task_mod.deactivate_stale_listings(
|
|
db, run_id=4, listing_source="yandex", ttl_days=30, segments=["vtorichka"]
|
|
) # type: ignore[arg-type]
|
|
_stmt, params = db.executed[0]
|
|
assert params is not None
|
|
assert params["listing_source"] == "yandex"
|
|
|
|
|
|
def test_fresh_listing_zero_rowcount(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
marked: dict[str, Any] = {}
|
|
monkeypatch.setattr(
|
|
task_mod.runs_mod,
|
|
"mark_done",
|
|
lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)),
|
|
)
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
|
|
db = _FakeDB(rowcount=0)
|
|
out = task_mod.deactivate_stale_listings(
|
|
db, run_id=5, listing_source="cian", ttl_days=30, segments=["vtorichka"]
|
|
) # type: ignore[arg-type]
|
|
assert out == {"deactivated": 0}
|
|
assert db.committed is True
|
|
assert marked["counters"] == {"deactivated": 0}
|
|
|
|
|
|
def test_segments_none_uses_all_segments_sql(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
|
|
db = _FakeDB(rowcount=3)
|
|
out = task_mod.deactivate_stale_listings(
|
|
db, run_id=6, listing_source="avito", ttl_days=10, segments=None
|
|
) # type: ignore[arg-type]
|
|
assert out == {"deactivated": 3}
|
|
_stmt, params = db.executed[0]
|
|
assert params is not None
|
|
assert "segments" not in params
|
|
assert params["listing_source"] == "avito"
|
|
assert params["ttl_days"] == 10
|
|
|
|
|
|
def test_avito_compat_wrapper_counter(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
marked: dict[str, Any] = {}
|
|
monkeypatch.setattr(
|
|
task_mod.runs_mod,
|
|
"mark_done",
|
|
lambda _db, run_id, counters: marked.update(run_id=run_id, counters=dict(counters)),
|
|
)
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_failed", lambda *a, **k: None)
|
|
db = _FakeDB(rowcount=42)
|
|
out = task_mod.deactivate_stale_avito_listings(db, run_id=7) # type: ignore[arg-type]
|
|
assert out == {"deactivated": 42}
|
|
assert db.committed is True
|
|
_stmt, params = db.executed[0]
|
|
assert params is not None
|
|
assert params["listing_source"] == "avito"
|
|
assert "segments" not in params
|
|
assert marked["run_id"] == 7
|
|
|
|
|
|
def test_failure_path_rollback_and_mark_failed(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
failed: dict[str, Any] = {}
|
|
monkeypatch.setattr(task_mod.runs_mod, "mark_done", lambda *a, **k: None)
|
|
monkeypatch.setattr(
|
|
task_mod.runs_mod,
|
|
"mark_failed",
|
|
lambda _db, run_id, err, counters: failed.update(run_id=run_id, err=err),
|
|
)
|
|
|
|
class _BoomDB(_FakeDB):
|
|
def execute(self, stmt: Any, params: dict[str, Any] | None = None) -> _FakeResult:
|
|
raise RuntimeError("db exploded")
|
|
|
|
db = _BoomDB()
|
|
with pytest.raises(RuntimeError, match="db exploded"):
|
|
task_mod.deactivate_stale_listings(
|
|
db, run_id=8, listing_source="yandex", ttl_days=30, segments=["vtorichka"]
|
|
) # type: ignore[arg-type]
|
|
assert db.rolled_back is True
|
|
assert failed["run_id"] == 8
|
|
|
|
|
|
def test_migration_115_exists() -> None:
|
|
assert _MIGRATION_115.is_file(), f"missing migration: {_MIGRATION_115}"
|
|
|
|
|
|
def test_migration_115_seeds_yandex_and_cian() -> None:
|
|
sql = _MIGRATION_115.read_text("utf-8")
|
|
assert "'deactivate_stale_yandex'" in sql
|
|
assert "'deactivate_stale_cian'" in sql
|
|
|
|
|
|
def test_migration_115_is_idempotent() -> None:
|
|
sql = _MIGRATION_115.read_text("utf-8")
|
|
assert "ON CONFLICT (source) DO NOTHING" in sql
|
|
|
|
|
|
def test_migration_115_enabled_true() -> None:
|
|
sql = _MIGRATION_115.read_text("utf-8")
|
|
assert "true" in sql
|
|
|
|
|
|
def test_migration_115_is_transactional() -> None:
|
|
sql = _MIGRATION_115.read_text("utf-8")
|
|
assert "BEGIN;" in sql
|
|
assert "COMMIT;" in sql
|
|
|
|
|
|
def test_migration_115_window_7_to_8_utc() -> None:
|
|
sql = _MIGRATION_115.read_text("utf-8")
|
|
assert re.search(r"\b7\b", sql), "window_start_hour 7 missing"
|
|
assert re.search(r"\b8\b", sql), "window_end_hour 8 missing"
|
|
|
|
|
|
def test_migration_115_segments_vtorichka_in_default_params() -> None:
|
|
sql = _MIGRATION_115.read_text("utf-8")
|
|
assert "vtorichka" in sql
|
|
|
|
|
|
def test_migration_115_ttl_30_days() -> None:
|
|
sql = _MIGRATION_115.read_text("utf-8")
|
|
assert "30" in sql
|
|
|
|
|
|
def test_migration_115_no_psycopg_trap() -> None:
|
|
sql = _MIGRATION_115.read_text("utf-8")
|
|
assert not re.search(r":\w+::", sql)
|