"""Runtime config loader for scraper delays. Backed by scraper_settings table with 60s in-process cache. """ from __future__ import annotations import logging import time from threading import Lock from sqlalchemy import text logger = logging.getLogger(__name__) # In-process cache: {source: (delay_sec, fetched_at_epoch)} _CACHE: dict[str, tuple[float, float]] = {} _CACHE_LOCK = Lock() _CACHE_TTL_SEC = 60.0 # Default fallback if DB row missing or DB unreachable _DEFAULT_DELAY_BY_SOURCE: dict[str, float] = { "yandex": 5.0, "yandex_detail": 5.0, "yandex_newbuilding": 5.0, "yandex_valuation": 5.0, } _GLOBAL_DEFAULT_DELAY = 5.0 # Yandex sub-scrapers all read the umbrella "yandex" key. # Map: scraper_name -> DB row key. _KEY_ALIASES: dict[str, str] = { "yandex": "yandex", "yandex_detail": "yandex", "yandex_newbuilding": "yandex", "yandex_realty_nb": "yandex", # BaseScraper.name used by YandexNewbuildingScraper "yandex_valuation": "yandex", # Non-Yandex sources resolve to their own keys for forward-compat # (no rows exist yet for them; fall back to default). } def _open_session() -> object: """Return a new SessionLocal context-manager instance. Deferred import avoids triggering Settings() validation at module import time (unit tests run without DATABASE_URL env var). Extracted as a named function so tests can patch it. """ from app.core.db import SessionLocal return SessionLocal() def get_scraper_delay(source: str) -> float: """Return the configured request_delay_sec for `source`. - Maps Yandex sub-scrapers to the umbrella 'yandex' key. - Cached 60s in-process to avoid per-instance DB query. - Any DB error -> returns the hardcoded default for that source. """ key = _KEY_ALIASES.get(source, source) now = time.time() with _CACHE_LOCK: cached = _CACHE.get(key) if cached is not None and now - cached[1] < _CACHE_TTL_SEC: return cached[0] try: with _open_session() as db: row = db.execute( text( "SELECT request_delay_sec FROM scraper_settings WHERE source = :s" ), {"s": key}, ).first() if row is not None: value = float(row[0]) else: value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY) except Exception as e: logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e) value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY) with _CACHE_LOCK: _CACHE[key] = (value, now) return value def invalidate_cache(source: str | None = None) -> None: """Clear the cache entry for `source`, or the whole cache if None. Called by admin endpoint after PUT to force a re-read on next get_scraper_delay(). """ with _CACHE_LOCK: if source is None: _CACHE.clear() else: key = _KEY_ALIASES.get(source, source) _CACHE.pop(key, None)