"""Tests for scraper_settings loader (cache + DB fallback + alias mapping + global delay). Covers: - Per-source value lookup + caching (main's tests) - Global delay max() logic (our additions) - Admin endpoint smoke """ import os os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") from unittest.mock import MagicMock, patch import pytest import app.services.scraper_settings as ss_mod from app.services import scraper_settings as ss from app.services.scraper_settings import get_scraper_delay, invalidate_cache # Patch target: _open_session is a module-level wrapper so tests can mock DB _PATCH_TARGET = "app.services.scraper_settings._open_session" # ── Fixtures ────────────────────────────────────────────────────────────────── @pytest.fixture(autouse=True) def _reset_cache(): ss._CACHE.clear() yield ss._CACHE.clear() # ── Helpers ─────────────────────────────────────────────────────────────────── def _mock_session_uniform(row_value: float | None) -> tuple[MagicMock, MagicMock]: """Mock that returns the same value for any key (context-manager style). Used by tests that don't care about multi-key distinction. Global key will also return the same value, so max() returns row_value. """ session = MagicMock() session.__enter__ = MagicMock(return_value=session) session.__exit__ = MagicMock(return_value=False) if row_value is None: session.execute.return_value.first.return_value = None else: session.execute.return_value.first.return_value = (row_value,) mock_open = MagicMock(return_value=session) return mock_open, session def _make_mock_session(values: dict[str, float | None]) -> MagicMock: """Mock сессии БД с фиксированными значениями per source key. Поддерживает разные значения для разных ключей — используется в тестах global delay логики. """ mock_db = MagicMock() def fake_execute(sql, params): key = params["s"] result = MagicMock() val = values.get(key) result.first.return_value = (val,) if val is not None else None return result mock_db.execute = fake_execute mock_db.__enter__ = MagicMock(return_value=mock_db) mock_db.__exit__ = MagicMock(return_value=False) return mock_db # ── Main's tests: core cache + alias + fallback behaviour ───────────────────── def test_returns_db_value_for_yandex(): """DB value 7.5, global also 7.5 → max(7.5, 7.5) = 7.5.""" mock_open, _ = _mock_session_uniform(7.5) with patch(_PATCH_TARGET, mock_open): assert ss.get_scraper_delay("yandex") == 7.5 def test_yandex_subkeys_map_to_yandex_umbrella(): """yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row.""" mock_open, _session = _mock_session_uniform(8.0) with patch(_PATCH_TARGET, mock_open): assert ss.get_scraper_delay("yandex_detail") == 8.0 assert ss.get_scraper_delay("yandex_newbuilding") == 8.0 assert ss.get_scraper_delay("yandex_valuation") == 8.0 # First call fetches yandex + global (2 DB calls). Remaining 2 subkey calls # map to "yandex" (already cached) and "global" (already cached) → 0 more. assert mock_open.call_count == 2 def test_cache_hit_within_ttl(): """After first call both yandex+global are cached; repeats hit cache only.""" mock_open, _session = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): ss.get_scraper_delay("yandex") ss.get_scraper_delay("yandex") ss.get_scraper_delay("yandex") # First call: 2 DB hits (yandex + global). Subsequent: 0. assert mock_open.call_count == 2 def test_cache_expires_after_ttl(): """Force-age yandex key — next call re-fetches yandex only (global still fresh).""" mock_open, _session = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): ss.get_scraper_delay("yandex") # +2 calls (yandex + global) # Force-age the yandex entry with ss._CACHE_LOCK: v, t = ss._CACHE["yandex"] ss._CACHE["yandex"] = (v, t - 999) ss.get_scraper_delay("yandex") # +1 call (yandex stale; global still fresh) assert mock_open.call_count == 3 def test_db_missing_row_returns_default(): """Missing yandex row → _DEFAULT_DELAY_BY_SOURCE. Global None → 0.0. max = default.""" mock_open, _ = _mock_session_uniform(None) with patch(_PATCH_TARGET, mock_open): assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"] def test_db_error_returns_default(): """DB error → hardcoded default. Global error → 0.0. max = default.""" bad_session = MagicMock() bad_session.__enter__ = MagicMock(return_value=bad_session) bad_session.__exit__ = MagicMock(return_value=False) bad_session.execute.side_effect = RuntimeError("db down") bad_open = MagicMock(return_value=bad_session) with patch(_PATCH_TARGET, bad_open): assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"] def test_unknown_source_uses_global_default(): """Unknown source, DB row None → _GLOBAL_DEFAULT_DELAY. Global None → 0.0. max = 5.0.""" mock_open, _ = _mock_session_uniform(None) with patch(_PATCH_TARGET, mock_open): assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY def test_invalidate_cache_specific_source(): """invalidate_cache(yandex) drops yandex entry; next call re-fetches yandex only.""" mock_open, _ = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): ss.get_scraper_delay("yandex") # +2 (yandex + global) ss.invalidate_cache("yandex") # clears yandex key ss.get_scraper_delay("yandex") # +1 (yandex stale; global still cached) assert mock_open.call_count == 3 def test_invalidate_cache_all(): """invalidate_cache() clears all entries; next call re-fetches both.""" mock_open, _ = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): ss.get_scraper_delay("yandex") # +2 (yandex + global) ss.invalidate_cache() # clears everything ss.get_scraper_delay("yandex") # +2 (both stale) assert mock_open.call_count == 4 def test_invalidate_alias_invalidates_umbrella(): """invalidate_cache('yandex_detail') clears the 'yandex' umbrella key.""" mock_open, _ = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): ss.get_scraper_delay("yandex_detail") # +2 (yandex + global) ss.invalidate_cache("yandex_detail") # clears yandex (via alias) ss.get_scraper_delay("yandex_detail") # +1 (yandex stale; global fresh) assert mock_open.call_count == 3 # ── Our additions: global delay max() logic ─────────────────────────────────── def test_global_delay_takes_precedence_when_higher(monkeypatch): """global=10 > avito=7 → returns 10.""" invalidate_cache() mock_db = _make_mock_session({"avito": 7.0, "global": 10.0}) monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) delay = get_scraper_delay("avito") assert delay == 10.0 def test_per_source_takes_precedence_when_higher(monkeypatch): """avito=7 > global=2 → returns 7.""" invalidate_cache() mock_db = _make_mock_session({"avito": 7.0, "global": 2.0}) monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) delay = get_scraper_delay("avito") assert delay == 7.0 def test_global_zero_means_no_floor(monkeypatch): """global=0 → only per-source value applies.""" invalidate_cache() mock_db = _make_mock_session({"cian": 5.0, "global": 0.0}) monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) delay = get_scraper_delay("cian") assert delay == 5.0 def test_missing_per_source_row_uses_class_default(monkeypatch): """Missing source row → _DEFAULT_DELAY_BY_SOURCE. global=0 → no floor.""" invalidate_cache() mock_db = _make_mock_session({"domklik": None, "global": 0.0}) monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) delay = get_scraper_delay("domklik") assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["domklik"] def test_global_missing_row_treated_as_zero(monkeypatch): """Missing 'global' row → fallback 0.0, avito=7.0 → max(7.0, 0.0) = 7.0.""" invalidate_cache() mock_db = _make_mock_session({"avito": 7.0, "global": None}) monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) delay = get_scraper_delay("avito") assert delay == 7.0 def test_db_error_falls_back_to_default_global_logic(monkeypatch): """DB error → graceful fallback, no crash. avito default=7.0, global error→0.0.""" invalidate_cache() def broken_open(): raise RuntimeError("DB is down") monkeypatch.setattr(ss_mod, "_open_session", broken_open) delay = get_scraper_delay("avito") assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["avito"] def test_invalidate_cache_single_source_global_logic(monkeypatch): """invalidate_cache(source) removes only that source from cache.""" invalidate_cache() mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0}) monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) get_scraper_delay("cian") get_scraper_delay("avito") invalidate_cache("cian") with ss_mod._CACHE_LOCK: assert "cian" not in ss_mod._CACHE assert "avito" in ss_mod._CACHE def test_invalidate_cache_all_global_logic(monkeypatch): """invalidate_cache() without args clears the entire cache.""" invalidate_cache() mock_db = _make_mock_session({"avito": 7.0, "global": 0.0}) monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) get_scraper_delay("avito") invalidate_cache() with ss_mod._CACHE_LOCK: assert len(ss_mod._CACHE) == 0 # ── Admin API endpoint smoke ─────────────────────────────────────────────────── def test_list_scraper_settings_endpoint(monkeypatch): """GET /scraper-settings mounts and returns a list including 'global'.""" import sys # Stub weasyprint (not on CI) _wp = MagicMock() sys.modules.setdefault("weasyprint", _wp) from fastapi import FastAPI from fastapi.testclient import TestClient from app.api.v1 import admin as admin_module from app.core.db import get_db app = FastAPI() app.include_router(admin_module.router, prefix="/api/v1/admin") fake_rows = [ {"source": "avito", "request_delay_sec": 7.0, "description": "Avito", "updated_at": None}, {"source": "global", "request_delay_sec": 0.0, "description": "Global", "updated_at": None}, ] mock_db = MagicMock() mock_db.execute.return_value.mappings.return_value.all.return_value = fake_rows app.dependency_overrides[get_db] = lambda: mock_db client = TestClient(app) r = client.get("/api/v1/admin/scraper-settings") assert r.status_code == 200 data = r.json() assert "settings" in data sources = [s["source"] for s in data["settings"]] assert "global" in sources assert "avito" in sources