gendesign/tradein-mvp/backend/tests/test_scraper_settings.py

"""Tests for scraper_settings loader (cache + DB fallback + alias mapping + global delay).

Covers:
  - Per-source value lookup + caching (main's tests)
  - Global delay max() logic (our additions)
  - Admin endpoint smoke
"""

import os

os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")

from unittest.mock import MagicMock, patch

import pytest

import app.services.scraper_settings as ss_mod
from app.services import scraper_settings as ss
from app.services.scraper_settings import get_scraper_delay, invalidate_cache

# Patch target: _open_session is a module-level wrapper so tests can mock DB
_PATCH_TARGET = "app.services.scraper_settings._open_session"


# ── Fixtures ──────────────────────────────────────────────────────────────────


@pytest.fixture(autouse=True)
def _reset_cache():
    ss._CACHE.clear()
    yield
    ss._CACHE.clear()


# ── Helpers ───────────────────────────────────────────────────────────────────


def _mock_session_uniform(row_value: float | None) -> tuple[MagicMock, MagicMock]:
    """Mock that returns the same value for any key (context-manager style).

    Used by tests that don't care about multi-key distinction.
    Global key will also return the same value, so max() returns row_value.
    """
    session = MagicMock()
    session.__enter__ = MagicMock(return_value=session)
    session.__exit__ = MagicMock(return_value=False)
    if row_value is None:
        session.execute.return_value.first.return_value = None
    else:
        session.execute.return_value.first.return_value = (row_value,)
    mock_open = MagicMock(return_value=session)
    return mock_open, session


def _make_mock_session(values: dict[str, float | None]) -> MagicMock:
    """Mock сессии БД с фиксированными значениями per source key.

    Поддерживает разные значения для разных ключей — используется в тестах
    global delay логики.
    """
    mock_db = MagicMock()

    def fake_execute(sql, params):
        key = params["s"]
        result = MagicMock()
        val = values.get(key)
        result.first.return_value = (val,) if val is not None else None
        return result

    mock_db.execute = fake_execute
    mock_db.__enter__ = MagicMock(return_value=mock_db)
    mock_db.__exit__ = MagicMock(return_value=False)
    return mock_db


# ── Main's tests: core cache + alias + fallback behaviour ─────────────────────


def test_returns_db_value_for_yandex():
    """DB value 7.5, global also 7.5 → max(7.5, 7.5) = 7.5."""
    mock_open, _ = _mock_session_uniform(7.5)
    with patch(_PATCH_TARGET, mock_open):
        assert ss.get_scraper_delay("yandex") == 7.5


def test_yandex_subkeys_map_to_yandex_umbrella():
    """yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row."""
    mock_open, _session = _mock_session_uniform(8.0)
    with patch(_PATCH_TARGET, mock_open):
        assert ss.get_scraper_delay("yandex_detail") == 8.0
        assert ss.get_scraper_delay("yandex_newbuilding") == 8.0
        assert ss.get_scraper_delay("yandex_valuation") == 8.0
    # First call fetches yandex + global (2 DB calls). Remaining 2 subkey calls
    # map to "yandex" (already cached) and "global" (already cached) → 0 more.
    assert mock_open.call_count == 2


def test_cache_hit_within_ttl():
    """After first call both yandex+global are cached; repeats hit cache only."""
    mock_open, _session = _mock_session_uniform(6.0)
    with patch(_PATCH_TARGET, mock_open):
        ss.get_scraper_delay("yandex")
        ss.get_scraper_delay("yandex")
        ss.get_scraper_delay("yandex")
    # First call: 2 DB hits (yandex + global). Subsequent: 0.
    assert mock_open.call_count == 2


def test_cache_expires_after_ttl():
    """Force-age yandex key — next call re-fetches yandex only (global still fresh)."""
    mock_open, _session = _mock_session_uniform(6.0)
    with patch(_PATCH_TARGET, mock_open):
        ss.get_scraper_delay("yandex")  # +2 calls (yandex + global)
        # Force-age the yandex entry
        with ss._CACHE_LOCK:
            v, t = ss._CACHE["yandex"]
            ss._CACHE["yandex"] = (v, t - 999)
        ss.get_scraper_delay("yandex")  # +1 call (yandex stale; global still fresh)
    assert mock_open.call_count == 3


def test_db_missing_row_returns_default():
    """Missing yandex row → _DEFAULT_DELAY_BY_SOURCE. Global None → 0.0. max = default."""
    mock_open, _ = _mock_session_uniform(None)
    with patch(_PATCH_TARGET, mock_open):
        assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]


def test_db_error_returns_default():
    """DB error → hardcoded default. Global error → 0.0. max = default."""
    bad_session = MagicMock()
    bad_session.__enter__ = MagicMock(return_value=bad_session)
    bad_session.__exit__ = MagicMock(return_value=False)
    bad_session.execute.side_effect = RuntimeError("db down")
    bad_open = MagicMock(return_value=bad_session)
    with patch(_PATCH_TARGET, bad_open):
        assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]


def test_unknown_source_uses_global_default():
    """Unknown source, DB row None → _GLOBAL_DEFAULT_DELAY. Global None → 0.0. max = 5.0."""
    mock_open, _ = _mock_session_uniform(None)
    with patch(_PATCH_TARGET, mock_open):
        assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY


def test_invalidate_cache_specific_source():
    """invalidate_cache(yandex) drops yandex entry; next call re-fetches yandex only."""
    mock_open, _ = _mock_session_uniform(6.0)
    with patch(_PATCH_TARGET, mock_open):
        ss.get_scraper_delay("yandex")  # +2 (yandex + global)
        ss.invalidate_cache("yandex")  # clears yandex key
        ss.get_scraper_delay("yandex")  # +1 (yandex stale; global still cached)
    assert mock_open.call_count == 3


def test_invalidate_cache_all():
    """invalidate_cache() clears all entries; next call re-fetches both."""
    mock_open, _ = _mock_session_uniform(6.0)
    with patch(_PATCH_TARGET, mock_open):
        ss.get_scraper_delay("yandex")  # +2 (yandex + global)
        ss.invalidate_cache()  # clears everything
        ss.get_scraper_delay("yandex")  # +2 (both stale)
    assert mock_open.call_count == 4


def test_invalidate_alias_invalidates_umbrella():
    """invalidate_cache('yandex_detail') clears the 'yandex' umbrella key."""
    mock_open, _ = _mock_session_uniform(6.0)
    with patch(_PATCH_TARGET, mock_open):
        ss.get_scraper_delay("yandex_detail")  # +2 (yandex + global)
        ss.invalidate_cache("yandex_detail")  # clears yandex (via alias)
        ss.get_scraper_delay("yandex_detail")  # +1 (yandex stale; global fresh)
    assert mock_open.call_count == 3


# ── Our additions: global delay max() logic ───────────────────────────────────


def test_global_delay_takes_precedence_when_higher(monkeypatch):
    """global=10 > avito=7 → returns 10."""
    invalidate_cache()

    mock_db = _make_mock_session({"avito": 7.0, "global": 10.0})
    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)

    delay = get_scraper_delay("avito")
    assert delay == 10.0


def test_per_source_takes_precedence_when_higher(monkeypatch):
    """avito=7 > global=2 → returns 7."""
    invalidate_cache()

    mock_db = _make_mock_session({"avito": 7.0, "global": 2.0})
    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)

    delay = get_scraper_delay("avito")
    assert delay == 7.0


def test_global_zero_means_no_floor(monkeypatch):
    """global=0 → only per-source value applies."""
    invalidate_cache()

    mock_db = _make_mock_session({"cian": 5.0, "global": 0.0})
    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)

    delay = get_scraper_delay("cian")
    assert delay == 5.0


def test_missing_per_source_row_uses_class_default(monkeypatch):
    """Missing source row → _DEFAULT_DELAY_BY_SOURCE. global=0 → no floor."""
    invalidate_cache()

    mock_db = _make_mock_session({"domklik": None, "global": 0.0})
    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)

    delay = get_scraper_delay("domklik")
    assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["domklik"]


def test_global_missing_row_treated_as_zero(monkeypatch):
    """Missing 'global' row → fallback 0.0, avito=7.0 → max(7.0, 0.0) = 7.0."""
    invalidate_cache()

    mock_db = _make_mock_session({"avito": 7.0, "global": None})
    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)

    delay = get_scraper_delay("avito")
    assert delay == 7.0


def test_db_error_falls_back_to_default_global_logic(monkeypatch):
    """DB error → graceful fallback, no crash. avito default=7.0, global error→0.0."""
    invalidate_cache()

    def broken_open():
        raise RuntimeError("DB is down")

    monkeypatch.setattr(ss_mod, "_open_session", broken_open)

    delay = get_scraper_delay("avito")
    assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["avito"]


def test_invalidate_cache_single_source_global_logic(monkeypatch):
    """invalidate_cache(source) removes only that source from cache."""
    invalidate_cache()

    mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0})
    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)

    get_scraper_delay("cian")
    get_scraper_delay("avito")

    invalidate_cache("cian")

    with ss_mod._CACHE_LOCK:
        assert "cian" not in ss_mod._CACHE
        assert "avito" in ss_mod._CACHE


def test_invalidate_cache_all_global_logic(monkeypatch):
    """invalidate_cache() without args clears the entire cache."""
    invalidate_cache()

    mock_db = _make_mock_session({"avito": 7.0, "global": 0.0})
    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)

    get_scraper_delay("avito")
    invalidate_cache()

    with ss_mod._CACHE_LOCK:
        assert len(ss_mod._CACHE) == 0


# ── Admin API endpoint smoke ───────────────────────────────────────────────────


def test_list_scraper_settings_endpoint(monkeypatch):
    """GET /scraper-settings mounts and returns a list including 'global'."""
    import sys

    # Stub weasyprint (not on CI)
    _wp = MagicMock()
    sys.modules.setdefault("weasyprint", _wp)

    from fastapi import FastAPI
    from fastapi.testclient import TestClient

    from app.api.v1 import admin as admin_module
    from app.core.db import get_db

    app = FastAPI()
    app.include_router(admin_module.router, prefix="/api/v1/admin")

    fake_rows = [
        {"source": "avito", "request_delay_sec": 7.0, "description": "Avito", "updated_at": None},
        {"source": "global", "request_delay_sec": 0.0, "description": "Global", "updated_at": None},
    ]

    mock_db = MagicMock()
    mock_db.execute.return_value.mappings.return_value.all.return_value = fake_rows
    app.dependency_overrides[get_db] = lambda: mock_db

    client = TestClient(app)
    r = client.get("/api/v1/admin/scraper-settings")
    assert r.status_code == 200
    data = r.json()
    assert "settings" in data
    sources = [s["source"] for s in data["settings"]]
    assert "global" in sources
    assert "avito" in sources