gendesign/tradein-mvp/backend/app/services/scraper_settings.py

"""scraper_settings.py — загрузка задержек парсеров из таблицы scraper_settings.

Кеш в памяти с TTL 60 секунд — не нагружает БД при каждом запросе.
invalidate_cache() вызывается из admin API PUT для немедленного применения.

Ключевая логика:
  get_scraper_delay(source) = max(per_source_delay, global_delay)
  Строка source='global' задаёт нижнюю планку для ВСЕХ парсеров.
  Если global=0 — используется только per-source значение.
"""

from __future__ import annotations

import logging
import time
from threading import Lock

from sqlalchemy import text

logger = logging.getLogger(__name__)

# Специальный ключ глобальной задержки (строка в scraper_settings с этим source).
_GLOBAL_KEY = "global"

# In-process cache: {source: (delay_sec, fetched_at_epoch)}
_CACHE: dict[str, tuple[float, float]] = {}
_CACHE_LOCK = Lock()
_CACHE_TTL_SEC = 60.0

# Default fallback if DB row missing or DB unreachable
_DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
    "avito": 7.0,
    "cian": 5.0,
    "domklik": 8.0,
    "yandex": 5.0,
    "yandex_detail": 5.0,
    "yandex_newbuilding": 5.0,
    "yandex_valuation": 5.0,
    "domrf": 5.0,
    "rosreestr": 5.0,
}
_GLOBAL_DEFAULT_DELAY = 5.0

# Yandex sub-scrapers all read the umbrella "yandex" key.
# Map: scraper_name -> DB row key.
_KEY_ALIASES: dict[str, str] = {
    "yandex": "yandex",
    "yandex_detail": "yandex",
    "yandex_newbuilding": "yandex",
    "yandex_realty_nb": "yandex",  # BaseScraper.name used by YandexNewbuildingScraper
    "yandex_valuation": "yandex",
    # Non-Yandex sources resolve to their own keys for forward-compat
    # (no rows exist yet for them; fall back to default).
}


def _open_session() -> object:
    """Return a new SessionLocal context-manager instance.

    Deferred import avoids triggering Settings() validation at module import
    time (unit tests run without DATABASE_URL env var).
    Extracted as a named function so tests can patch it.
    """
    from app.core.db import SessionLocal

    return SessionLocal()


def _get_setting_cached(key: str) -> float:
    """Cache-aware read of one source key from scraper_settings.

    Returns request_delay_sec from DB (with TTL=60s cache).
    Missing row: falls back to _DEFAULT_DELAY_BY_SOURCE, or 0.0 for _GLOBAL_KEY
    (meaning no global floor is applied when the row hasn't been seeded yet).
    """
    now = time.time()
    with _CACHE_LOCK:
        cached = _CACHE.get(key)
        if cached is not None and now - cached[1] < _CACHE_TTL_SEC:
            return cached[0]

    try:
        with _open_session() as db:
            row = db.execute(
                text("SELECT request_delay_sec FROM scraper_settings WHERE source = :s"),
                {"s": key},
            ).first()
        if row is not None:
            value = float(row[0])
        elif key == _GLOBAL_KEY:
            # Global row not yet seeded — don't apply floor.
            value = 0.0
        else:
            value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
    except Exception as e:
        logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
        value = (
            0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
        )

    with _CACHE_LOCK:
        _CACHE[key] = (value, now)
    return value


def get_scraper_delay(source: str) -> float:
    """Return the configured request_delay_sec for `source`.

    Effective delay = max(per_source, global).
    - Maps Yandex sub-scrapers to the umbrella 'yandex' key.
    - Cached 60s in-process to avoid per-instance DB query.
    - source='global' sets a floor across all scrapers (0 = disabled).
    - Any DB error -> returns the hardcoded default for that source.
    """
    key = _KEY_ALIASES.get(source, source)
    per_source = _get_setting_cached(key)
    global_value = _get_setting_cached(_GLOBAL_KEY)
    return max(per_source, global_value)


def invalidate_cache(source: str | None = None) -> None:
    """Clear the cache entry for `source`, or the whole cache if None.

    Called by admin endpoint after PUT to force a re-read on next get_scraper_delay().
    """
    with _CACHE_LOCK:
        if source is None:
            _CACHE.clear()
            logger.info("scraper_settings: cache cleared (all sources)")
        else:
            key = _KEY_ALIASES.get(source, source)
            _CACHE.pop(key, None)
            # Also clear the source itself if it differed from key (alias case).
            if key != source:
                _CACHE.pop(source, None)
            logger.info("scraper_settings: cache cleared for source=%s", source)