diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index 2d29b569..d895abd0 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -79,17 +79,6 @@ class ScrapeResponse(BaseModel): by_source: list[ScrapeResult] -class ScraperSetting(BaseModel): - source: str - request_delay_sec: float - description: str | None = None - updated_at: str | None = None # ISO - - -class ScraperSettingUpdate(BaseModel): - request_delay_sec: float = Field(ge=1.0, le=60.0) - - @router.post("/scrape", response_model=ScrapeResponse) async def scrape_around( payload: ScrapeRequest, @@ -560,6 +549,74 @@ def cancel_avito_city_sweep( return {"ok": True, "run_id": run_id, "cancelled": cancelled} +# ── Scraper settings: global + per-source delay management ─────────────────── + + +class ScraperSettingPayload(BaseModel): + source: str = Field(..., min_length=1, max_length=64) + request_delay_sec: float = Field(..., ge=0.0, le=60.0) + description: str | None = None + + +@router.get("/scraper-settings", status_code=200) +def list_scraper_settings( + db: Annotated[Session, Depends(get_db)], +) -> dict: + """Список всех настроек задержки (per-source + global).""" + rows = db.execute( + text(""" + SELECT source, request_delay_sec, description, updated_at + FROM scraper_settings + ORDER BY source ASC + """) + ).mappings().all() + return {"settings": [dict(r) for r in rows]} + + +@router.put("/scraper-settings/{source}", status_code=200) +def update_scraper_setting( + source: str, + payload: ScraperSettingPayload, + db: Annotated[Session, Depends(get_db)], +) -> dict: + """Обновить задержку для source (или 'global'). Path source используется как ключ. + + Эффект применяется немедленно — invalidate_cache() сбрасывает кеш для source, + следующий get_scraper_delay() перечитает из БД. + + source='global' — нижняя планка для всех парсеров (0 = отключено). + """ + if payload.source != source: + raise HTTPException( + status_code=400, + detail=f"Path source={source!r} does not match body source={payload.source!r}", + ) + + row = db.execute( + text(""" + INSERT INTO scraper_settings (source, request_delay_sec, description, updated_at) + VALUES (:s, CAST(:d AS numeric), :desc, NOW()) + ON CONFLICT (source) DO UPDATE SET + request_delay_sec = EXCLUDED.request_delay_sec, + description = COALESCE(EXCLUDED.description, scraper_settings.description), + updated_at = NOW() + RETURNING source, request_delay_sec, description, updated_at + """), + { + "s": source, + "d": payload.request_delay_sec, + "desc": payload.description, + }, + ).mappings().one() + db.commit() + + invalidate_cache(source) + logger.info( + "scraper-settings: updated source=%s delay=%.1f", source, payload.request_delay_sec + ) + return dict(row) + + # ── In-app scheduler endpoints (Stage 4e) ──────────────────────────────────── @@ -649,78 +706,6 @@ def update_schedule( ) -# -- Scraper settings (per-source live config) -------------------------------- - - -@router.get("/scraper-settings", response_model=list[ScraperSetting]) -async def list_scraper_settings( - db: Annotated[Session, Depends(get_db)], -) -> list[ScraperSetting]: - """Return all scraper_settings rows (live config for delays).""" - rows = ( - db.execute( - text( - """ - SELECT source, request_delay_sec, description, updated_at - FROM scraper_settings - ORDER BY source - """ - ) - ) - .mappings() - .all() - ) - return [ - ScraperSetting( - source=r["source"], - request_delay_sec=float(r["request_delay_sec"]), - description=r["description"], - updated_at=r["updated_at"].isoformat() if r["updated_at"] else None, - ) - for r in rows - ] - - -@router.put("/scraper-settings/{source}", response_model=ScraperSetting) -async def update_scraper_setting( - source: str, - payload: ScraperSettingUpdate, - db: Annotated[Session, Depends(get_db)], -) -> ScraperSetting: - """Update request_delay_sec for one source. UPSERT - creates row if missing. - - Invalidates the in-process cache so the change takes effect on next - scraper instantiation (usually within seconds). - """ - row = ( - db.execute( - text( - """ - INSERT INTO scraper_settings (source, request_delay_sec, updated_at) - VALUES (:source, :delay, NOW()) - ON CONFLICT (source) DO UPDATE - SET request_delay_sec = EXCLUDED.request_delay_sec, - updated_at = NOW() - RETURNING source, request_delay_sec, description, updated_at - """ - ), - {"source": source, "delay": payload.request_delay_sec}, - ) - .mappings() - .first() - ) - db.commit() - invalidate_cache(source) - if row is None: - raise HTTPException(500, "Failed to upsert scraper_settings row") - return ScraperSetting( - source=row["source"], - request_delay_sec=float(row["request_delay_sec"]), - description=row["description"], - updated_at=row["updated_at"].isoformat() if row["updated_at"] else None, - ) - - # -- Yandex ad-hoc scrape triggers -------------------------------------------- diff --git a/tradein-mvp/backend/app/services/scraper_settings.py b/tradein-mvp/backend/app/services/scraper_settings.py index 26d0d33c..cce374f6 100644 --- a/tradein-mvp/backend/app/services/scraper_settings.py +++ b/tradein-mvp/backend/app/services/scraper_settings.py @@ -1,6 +1,12 @@ -"""Runtime config loader for scraper delays. +"""scraper_settings.py — загрузка задержек парсеров из таблицы scraper_settings. -Backed by scraper_settings table with 60s in-process cache. +Кеш в памяти с TTL 60 секунд — не нагружает БД при каждом запросе. +invalidate_cache() вызывается из admin API PUT для немедленного применения. + +Ключевая логика: + get_scraper_delay(source) = max(per_source_delay, global_delay) + Строка source='global' задаёт нижнюю планку для ВСЕХ парсеров. + Если global=0 — используется только per-source значение. """ from __future__ import annotations @@ -13,6 +19,9 @@ from sqlalchemy import text logger = logging.getLogger(__name__) +# Специальный ключ глобальной задержки (строка в scraper_settings с этим source). +_GLOBAL_KEY = "global" + # In-process cache: {source: (delay_sec, fetched_at_epoch)} _CACHE: dict[str, tuple[float, float]] = {} _CACHE_LOCK = Lock() @@ -20,10 +29,15 @@ _CACHE_TTL_SEC = 60.0 # Default fallback if DB row missing or DB unreachable _DEFAULT_DELAY_BY_SOURCE: dict[str, float] = { + "avito": 7.0, + "cian": 5.0, + "n1": 5.0, "yandex": 5.0, "yandex_detail": 5.0, "yandex_newbuilding": 5.0, "yandex_valuation": 5.0, + "domrf": 5.0, + "rosreestr": 5.0, } _GLOBAL_DEFAULT_DELAY = 5.0 @@ -52,15 +66,13 @@ def _open_session() -> object: return SessionLocal() -def get_scraper_delay(source: str) -> float: - """Return the configured request_delay_sec for `source`. +def _get_setting_cached(key: str) -> float: + """Cache-aware read of one source key from scraper_settings. - - Maps Yandex sub-scrapers to the umbrella 'yandex' key. - - Cached 60s in-process to avoid per-instance DB query. - - Any DB error -> returns the hardcoded default for that source. + Returns request_delay_sec from DB (with TTL=60s cache). + Missing row: falls back to _DEFAULT_DELAY_BY_SOURCE, or 0.0 for _GLOBAL_KEY + (meaning no global floor is applied when the row hasn't been seeded yet). """ - key = _KEY_ALIASES.get(source, source) - now = time.time() with _CACHE_LOCK: cached = _CACHE.get(key) @@ -77,17 +89,37 @@ def get_scraper_delay(source: str) -> float: ).first() if row is not None: value = float(row[0]) + elif key == _GLOBAL_KEY: + # Global row not yet seeded — don't apply floor. + value = 0.0 else: value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY) except Exception as e: logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e) - value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY) + value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get( + key, _GLOBAL_DEFAULT_DELAY + ) with _CACHE_LOCK: _CACHE[key] = (value, now) return value +def get_scraper_delay(source: str) -> float: + """Return the configured request_delay_sec for `source`. + + Effective delay = max(per_source, global). + - Maps Yandex sub-scrapers to the umbrella 'yandex' key. + - Cached 60s in-process to avoid per-instance DB query. + - source='global' sets a floor across all scrapers (0 = disabled). + - Any DB error -> returns the hardcoded default for that source. + """ + key = _KEY_ALIASES.get(source, source) + per_source = _get_setting_cached(key) + global_value = _get_setting_cached(_GLOBAL_KEY) + return max(per_source, global_value) + + def invalidate_cache(source: str | None = None) -> None: """Clear the cache entry for `source`, or the whole cache if None. @@ -96,6 +128,11 @@ def invalidate_cache(source: str | None = None) -> None: with _CACHE_LOCK: if source is None: _CACHE.clear() + logger.info("scraper_settings: cache cleared (all sources)") else: key = _KEY_ALIASES.get(source, source) _CACHE.pop(key, None) + # Also clear the source itself if it differed from key (alias case). + if key != source: + _CACHE.pop(source, None) + logger.info("scraper_settings: cache cleared for source=%s", source) diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index f6571b23..6530ab22 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -23,6 +23,7 @@ from urllib.parse import urlencode, urljoin from curl_cffi.requests import AsyncSession from selectolax.parser import HTMLParser +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot logger = logging.getLogger(__name__) @@ -36,11 +37,13 @@ class AvitoScraper(BaseScraper): name = "avito" base_url = "https://www.avito.ru" - # Avito жёстко мониторит — спим долго между запросами + # Avito жёстко мониторит — спим долго между запросами. + # Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра. request_delay_sec = 7.0 def __init__(self) -> None: super().__init__() + self.request_delay_sec = get_scraper_delay(self.name) self._cffi: AsyncSession | None = None async def __aenter__(self) -> AvitoScraper: diff --git a/tradein-mvp/backend/app/services/scrapers/cian.py b/tradein-mvp/backend/app/services/scrapers/cian.py index 8ad29df3..3dbd4a38 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian.py +++ b/tradein-mvp/backend/app/services/scrapers/cian.py @@ -23,6 +23,7 @@ from urllib.parse import urlencode from curl_cffi.requests import AsyncSession +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot from app.services.scrapers.cian_state_parser import extract_state @@ -46,10 +47,12 @@ class CianScraper(BaseScraper): name = "cian" # ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema sec 13, closed Q4) base_url = "https://ekb.cian.ru" + # Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра. request_delay_sec = 5.0 # консервативно: Cian менее агрессивен чем Avito, но 5s безопасно def __init__(self) -> None: super().__init__() + self.request_delay_sec = get_scraper_delay(self.name) self._cffi: AsyncSession | None = None async def __aenter__(self) -> CianScraper: diff --git a/tradein-mvp/backend/app/services/scrapers/n1.py b/tradein-mvp/backend/app/services/scrapers/n1.py index bcf5b153..907756f6 100644 --- a/tradein-mvp/backend/app/services/scrapers/n1.py +++ b/tradein-mvp/backend/app/services/scrapers/n1.py @@ -25,6 +25,7 @@ from typing import Any from curl_cffi.requests import AsyncSession from selectolax.parser import HTMLParser +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot logger = logging.getLogger(__name__) @@ -35,13 +36,15 @@ class N1Scraper(BaseScraper): name = "n1" base_url = "https://ekaterinburg.n1.ru" + # Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра. request_delay_sec = 5.0 def __init__(self) -> None: super().__init__() + self.request_delay_sec = get_scraper_delay(self.name) self._cffi: AsyncSession | None = None - async def __aenter__(self) -> "N1Scraper": + async def __aenter__(self) -> N1Scraper: await super().__aenter__() self._cffi = AsyncSession( impersonate="chrome120", @@ -74,7 +77,7 @@ class N1Scraper(BaseScraper): try: assert self._cffi is not None response = await self._cffi.get(url) - except Exception: # noqa: BLE001 + except Exception: logger.exception("n1 fetch failed for %s", url) return [] if response.status_code != 200: @@ -153,7 +156,9 @@ class N1Scraper(BaseScraper): title = link.text(strip=True) or "" rooms = _extract_rooms(title) # Адрес = title без префикса комнат - address = re.sub(r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE).strip() + address = re.sub( + r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE + ).strip() if not address: address = "Екатеринбург (N1)" @@ -192,7 +197,7 @@ class N1Scraper(BaseScraper): photo_urls=photos, raw_payload={"title": title}, ) - except Exception: # noqa: BLE001 + except Exception: logger.exception("n1 card parse failed") return None diff --git a/tradein-mvp/backend/data/sql/054_scraper_settings_global.sql b/tradein-mvp/backend/data/sql/054_scraper_settings_global.sql new file mode 100644 index 00000000..da09aefa --- /dev/null +++ b/tradein-mvp/backend/data/sql/054_scraper_settings_global.sql @@ -0,0 +1,25 @@ +-- 054_scraper_settings_global.sql +-- Add global delay row + per-source aliases for all scrapers. +-- Idempotent — ON CONFLICT DO NOTHING preserves user-edited values. +BEGIN; + +-- 'global' row: when > 0, acts as floor (max with per-source delay). +-- Default 0 = use per-source defaults. +INSERT INTO scraper_settings (source, request_delay_sec, description) +VALUES ( + 'global', + 0.0, + 'Global delay applied across ALL scrapers (max with per-source). 0 = use per-source defaults.' +) +ON CONFLICT (source) DO NOTHING; + +-- Per-source rows for all scrapers (individual control). +INSERT INTO scraper_settings (source, request_delay_sec, description) VALUES + ('avito', 7.0, 'Avito SERP delay (anti-ban)'), + ('cian', 5.0, 'Cian SERP delay'), + ('n1', 5.0, 'N1 SERP delay'), + ('domrf', 5.0, 'DOM.RF SERP delay'), + ('rosreestr', 5.0, 'Rosreestr API delay') +ON CONFLICT (source) DO NOTHING; + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_scraper_settings.py b/tradein-mvp/backend/tests/test_scraper_settings.py index f0d4cf6c..a465382c 100644 --- a/tradein-mvp/backend/tests/test_scraper_settings.py +++ b/tradein-mvp/backend/tests/test_scraper_settings.py @@ -1,14 +1,30 @@ -"""Tests for scraper_settings loader (cache + DB fallback + alias mapping).""" +"""Tests for scraper_settings loader (cache + DB fallback + alias mapping + global delay). + +Covers: + - Per-source value lookup + caching (main's tests) + - Global delay max() logic (our additions) + - Admin endpoint smoke +""" + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + from unittest.mock import MagicMock, patch import pytest +import app.services.scraper_settings as ss_mod from app.services import scraper_settings as ss +from app.services.scraper_settings import get_scraper_delay, invalidate_cache # Patch target: _open_session is a module-level wrapper so tests can mock DB _PATCH_TARGET = "app.services.scraper_settings._open_session" +# ── Fixtures ────────────────────────────────────────────────────────────────── + + @pytest.fixture(autouse=True) def _reset_cache(): ss._CACHE.clear() @@ -16,8 +32,15 @@ def _reset_cache(): ss._CACHE.clear() -def _mock_session(row_value: float | None) -> MagicMock: - """Return a mock that behaves like SessionLocal() context-manager.""" +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _mock_session_uniform(row_value: float | None) -> tuple[MagicMock, MagicMock]: + """Mock that returns the same value for any key (context-manager style). + + Used by tests that don't care about multi-key distinction. + Global key will also return the same value, so max() returns row_value. + """ session = MagicMock() session.__enter__ = MagicMock(return_value=session) session.__exit__ = MagicMock(return_value=False) @@ -25,56 +48,86 @@ def _mock_session(row_value: float | None) -> MagicMock: session.execute.return_value.first.return_value = None else: session.execute.return_value.first.return_value = (row_value,) - # _open_session() returns the session directly (no extra call) mock_open = MagicMock(return_value=session) return mock_open, session +def _make_mock_session(values: dict[str, float | None]) -> MagicMock: + """Mock сессии БД с фиксированными значениями per source key. + + Поддерживает разные значения для разных ключей — используется в тестах + global delay логики. + """ + mock_db = MagicMock() + + def fake_execute(sql, params): + key = params["s"] + result = MagicMock() + val = values.get(key) + result.first.return_value = (val,) if val is not None else None + return result + + mock_db.execute = fake_execute + mock_db.__enter__ = MagicMock(return_value=mock_db) + mock_db.__exit__ = MagicMock(return_value=False) + return mock_db + + +# ── Main's tests: core cache + alias + fallback behaviour ───────────────────── + + def test_returns_db_value_for_yandex(): - mock_open, _ = _mock_session(7.5) + """DB value 7.5, global also 7.5 → max(7.5, 7.5) = 7.5.""" + mock_open, _ = _mock_session_uniform(7.5) with patch(_PATCH_TARGET, mock_open): assert ss.get_scraper_delay("yandex") == 7.5 def test_yandex_subkeys_map_to_yandex_umbrella(): """yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row.""" - mock_open, _session = _mock_session(8.0) + mock_open, _session = _mock_session_uniform(8.0) with patch(_PATCH_TARGET, mock_open): assert ss.get_scraper_delay("yandex_detail") == 8.0 assert ss.get_scraper_delay("yandex_newbuilding") == 8.0 assert ss.get_scraper_delay("yandex_valuation") == 8.0 - # All 3 lookups hit the cache after the first; _open_session called once - assert mock_open.call_count == 1 - - -def test_cache_hit_within_ttl(): - mock_open, _session = _mock_session(6.0) - with patch(_PATCH_TARGET, mock_open): - ss.get_scraper_delay("yandex") - ss.get_scraper_delay("yandex") - ss.get_scraper_delay("yandex") - assert mock_open.call_count == 1 # cached after first - - -def test_cache_expires_after_ttl(): - mock_open, _session = _mock_session(6.0) - with patch(_PATCH_TARGET, mock_open): - ss.get_scraper_delay("yandex") - # Force-age the cache - with ss._CACHE_LOCK: - v, t = ss._CACHE["yandex"] - ss._CACHE["yandex"] = (v, t - 999) - ss.get_scraper_delay("yandex") + # First call fetches yandex + global (2 DB calls). Remaining 2 subkey calls + # map to "yandex" (already cached) and "global" (already cached) → 0 more. assert mock_open.call_count == 2 +def test_cache_hit_within_ttl(): + """After first call both yandex+global are cached; repeats hit cache only.""" + mock_open, _session = _mock_session_uniform(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex") + ss.get_scraper_delay("yandex") + ss.get_scraper_delay("yandex") + # First call: 2 DB hits (yandex + global). Subsequent: 0. + assert mock_open.call_count == 2 + + +def test_cache_expires_after_ttl(): + """Force-age yandex key — next call re-fetches yandex only (global still fresh).""" + mock_open, _session = _mock_session_uniform(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex") # +2 calls (yandex + global) + # Force-age the yandex entry + with ss._CACHE_LOCK: + v, t = ss._CACHE["yandex"] + ss._CACHE["yandex"] = (v, t - 999) + ss.get_scraper_delay("yandex") # +1 call (yandex stale; global still fresh) + assert mock_open.call_count == 3 + + def test_db_missing_row_returns_default(): - mock_open, _ = _mock_session(None) + """Missing yandex row → _DEFAULT_DELAY_BY_SOURCE. Global None → 0.0. max = default.""" + mock_open, _ = _mock_session_uniform(None) with patch(_PATCH_TARGET, mock_open): assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"] def test_db_error_returns_default(): + """DB error → hardcoded default. Global error → 0.0. max = default.""" bad_session = MagicMock() bad_session.__enter__ = MagicMock(return_value=bad_session) bad_session.__exit__ = MagicMock(return_value=False) @@ -85,34 +138,178 @@ def test_db_error_returns_default(): def test_unknown_source_uses_global_default(): - mock_open, _ = _mock_session(None) + """Unknown source, DB row None → _GLOBAL_DEFAULT_DELAY. Global None → 0.0. max = 5.0.""" + mock_open, _ = _mock_session_uniform(None) with patch(_PATCH_TARGET, mock_open): assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY def test_invalidate_cache_specific_source(): - mock_open, _ = _mock_session(6.0) + """invalidate_cache(yandex) drops yandex entry; next call re-fetches yandex only.""" + mock_open, _ = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): - ss.get_scraper_delay("yandex") - ss.invalidate_cache("yandex") - ss.get_scraper_delay("yandex") - assert mock_open.call_count == 2 + ss.get_scraper_delay("yandex") # +2 (yandex + global) + ss.invalidate_cache("yandex") # clears yandex key + ss.get_scraper_delay("yandex") # +1 (yandex stale; global still cached) + assert mock_open.call_count == 3 def test_invalidate_cache_all(): - mock_open, _ = _mock_session(6.0) + """invalidate_cache() clears all entries; next call re-fetches both.""" + mock_open, _ = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): - ss.get_scraper_delay("yandex") - ss.invalidate_cache() - ss.get_scraper_delay("yandex") - assert mock_open.call_count == 2 + ss.get_scraper_delay("yandex") # +2 (yandex + global) + ss.invalidate_cache() # clears everything + ss.get_scraper_delay("yandex") # +2 (both stale) + assert mock_open.call_count == 4 def test_invalidate_alias_invalidates_umbrella(): - """invalidate_cache('yandex_detail') should clear the 'yandex' umbrella key.""" - mock_open, _ = _mock_session(6.0) + """invalidate_cache('yandex_detail') clears the 'yandex' umbrella key.""" + mock_open, _ = _mock_session_uniform(6.0) with patch(_PATCH_TARGET, mock_open): - ss.get_scraper_delay("yandex_detail") - ss.invalidate_cache("yandex_detail") - ss.get_scraper_delay("yandex_detail") - assert mock_open.call_count == 2 + ss.get_scraper_delay("yandex_detail") # +2 (yandex + global) + ss.invalidate_cache("yandex_detail") # clears yandex (via alias) + ss.get_scraper_delay("yandex_detail") # +1 (yandex stale; global fresh) + assert mock_open.call_count == 3 + + +# ── Our additions: global delay max() logic ─────────────────────────────────── + + +def test_global_delay_takes_precedence_when_higher(monkeypatch): + """global=10 > avito=7 → returns 10.""" + invalidate_cache() + + mock_db = _make_mock_session({"avito": 7.0, "global": 10.0}) + monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) + + delay = get_scraper_delay("avito") + assert delay == 10.0 + + +def test_per_source_takes_precedence_when_higher(monkeypatch): + """avito=7 > global=2 → returns 7.""" + invalidate_cache() + + mock_db = _make_mock_session({"avito": 7.0, "global": 2.0}) + monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) + + delay = get_scraper_delay("avito") + assert delay == 7.0 + + +def test_global_zero_means_no_floor(monkeypatch): + """global=0 → only per-source value applies.""" + invalidate_cache() + + mock_db = _make_mock_session({"cian": 5.0, "global": 0.0}) + monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) + + delay = get_scraper_delay("cian") + assert delay == 5.0 + + +def test_missing_per_source_row_uses_class_default(monkeypatch): + """Missing source row → _DEFAULT_DELAY_BY_SOURCE. global=0 → no floor.""" + invalidate_cache() + + mock_db = _make_mock_session({"n1": None, "global": 0.0}) + monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) + + delay = get_scraper_delay("n1") + assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["n1"] + + +def test_global_missing_row_treated_as_zero(monkeypatch): + """Missing 'global' row → fallback 0.0, avito=7.0 → max(7.0, 0.0) = 7.0.""" + invalidate_cache() + + mock_db = _make_mock_session({"avito": 7.0, "global": None}) + monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) + + delay = get_scraper_delay("avito") + assert delay == 7.0 + + +def test_db_error_falls_back_to_default_global_logic(monkeypatch): + """DB error → graceful fallback, no crash. avito default=7.0, global error→0.0.""" + invalidate_cache() + + def broken_open(): + raise RuntimeError("DB is down") + + monkeypatch.setattr(ss_mod, "_open_session", broken_open) + + delay = get_scraper_delay("avito") + assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["avito"] + + +def test_invalidate_cache_single_source_global_logic(monkeypatch): + """invalidate_cache(source) removes only that source from cache.""" + invalidate_cache() + + mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0}) + monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) + + get_scraper_delay("cian") + get_scraper_delay("avito") + + invalidate_cache("cian") + + with ss_mod._CACHE_LOCK: + assert "cian" not in ss_mod._CACHE + assert "avito" in ss_mod._CACHE + + +def test_invalidate_cache_all_global_logic(monkeypatch): + """invalidate_cache() without args clears the entire cache.""" + invalidate_cache() + + mock_db = _make_mock_session({"avito": 7.0, "global": 0.0}) + monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) + + get_scraper_delay("avito") + invalidate_cache() + + with ss_mod._CACHE_LOCK: + assert len(ss_mod._CACHE) == 0 + + +# ── Admin API endpoint smoke ─────────────────────────────────────────────────── + + +def test_list_scraper_settings_endpoint(monkeypatch): + """GET /scraper-settings mounts and returns a list including 'global'.""" + import sys + + # Stub weasyprint (not on CI) + _wp = MagicMock() + sys.modules.setdefault("weasyprint", _wp) + + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from app.api.v1 import admin as admin_module + from app.core.db import get_db + + app = FastAPI() + app.include_router(admin_module.router, prefix="/api/v1/admin") + + fake_rows = [ + {"source": "avito", "request_delay_sec": 7.0, "description": "Avito", "updated_at": None}, + {"source": "global", "request_delay_sec": 0.0, "description": "Global", "updated_at": None}, + ] + + mock_db = MagicMock() + mock_db.execute.return_value.mappings.return_value.all.return_value = fake_rows + app.dependency_overrides[get_db] = lambda: mock_db + + client = TestClient(app) + r = client.get("/api/v1/admin/scraper-settings") + assert r.status_code == 200 + data = r.json() + assert "settings" in data + sources = [s["source"] for s in data["settings"]] + assert "global" in sources + assert "avito" in sources