"""scraper_settings.py — загрузка задержек парсеров из таблицы scraper_settings. Кеш в памяти с TTL 60 секунд — не нагружает БД при каждом запросе. invalidate_cache() вызывается из admin API PUT для немедленного применения. Ключевая логика: get_scraper_delay(source) = max(per_source_delay, global_delay) Строка source='global' задаёт нижнюю планку для ВСЕХ парсеров. Если global=0 — используется только per-source значение. """ from __future__ import annotations import logging import time from threading import Lock from sqlalchemy import text logger = logging.getLogger(__name__) # Специальный ключ глобальной задержки (строка в scraper_settings с этим source). _GLOBAL_KEY = "global" # In-process cache: {source: (delay_sec, fetched_at_epoch)} _CACHE: dict[str, tuple[float, float]] = {} _CACHE_LOCK = Lock() _CACHE_TTL_SEC = 60.0 # Default fallback if DB row missing or DB unreachable _DEFAULT_DELAY_BY_SOURCE: dict[str, float] = { "avito": 7.0, "cian": 5.0, "n1": 5.0, "yandex": 5.0, "yandex_detail": 5.0, "yandex_newbuilding": 5.0, "yandex_valuation": 5.0, "domrf": 5.0, "rosreestr": 5.0, } _GLOBAL_DEFAULT_DELAY = 5.0 # Yandex sub-scrapers all read the umbrella "yandex" key. # Map: scraper_name -> DB row key. _KEY_ALIASES: dict[str, str] = { "yandex": "yandex", "yandex_detail": "yandex", "yandex_newbuilding": "yandex", "yandex_realty_nb": "yandex", # BaseScraper.name used by YandexNewbuildingScraper "yandex_valuation": "yandex", # Non-Yandex sources resolve to their own keys for forward-compat # (no rows exist yet for them; fall back to default). } def _open_session() -> object: """Return a new SessionLocal context-manager instance. Deferred import avoids triggering Settings() validation at module import time (unit tests run without DATABASE_URL env var). Extracted as a named function so tests can patch it. """ from app.core.db import SessionLocal return SessionLocal() def _get_setting_cached(key: str) -> float: """Cache-aware read of one source key from scraper_settings. Returns request_delay_sec from DB (with TTL=60s cache). Missing row: falls back to _DEFAULT_DELAY_BY_SOURCE, or 0.0 for _GLOBAL_KEY (meaning no global floor is applied when the row hasn't been seeded yet). """ now = time.time() with _CACHE_LOCK: cached = _CACHE.get(key) if cached is not None and now - cached[1] < _CACHE_TTL_SEC: return cached[0] try: with _open_session() as db: row = db.execute( text( "SELECT request_delay_sec FROM scraper_settings WHERE source = :s" ), {"s": key}, ).first() if row is not None: value = float(row[0]) elif key == _GLOBAL_KEY: # Global row not yet seeded — don't apply floor. value = 0.0 else: value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY) except Exception as e: logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e) value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get( key, _GLOBAL_DEFAULT_DELAY ) with _CACHE_LOCK: _CACHE[key] = (value, now) return value def get_scraper_delay(source: str) -> float: """Return the configured request_delay_sec for `source`. Effective delay = max(per_source, global). - Maps Yandex sub-scrapers to the umbrella 'yandex' key. - Cached 60s in-process to avoid per-instance DB query. - source='global' sets a floor across all scrapers (0 = disabled). - Any DB error -> returns the hardcoded default for that source. """ key = _KEY_ALIASES.get(source, source) per_source = _get_setting_cached(key) global_value = _get_setting_cached(_GLOBAL_KEY) return max(per_source, global_value) def invalidate_cache(source: str | None = None) -> None: """Clear the cache entry for `source`, or the whole cache if None. Called by admin endpoint after PUT to force a re-read on next get_scraper_delay(). """ with _CACHE_LOCK: if source is None: _CACHE.clear() logger.info("scraper_settings: cache cleared (all sources)") else: key = _KEY_ALIASES.get(source, source) _CACHE.pop(key, None) # Also clear the source itself if it differed from key (alias case). if key != source: _CACHE.pop(source, None) logger.info("scraper_settings: cache cleared for source=%s", source)