e5fb209ced
Backend foundation for the Yandex admin scraper page (UI follows in next PR).
scraper_settings (mig 053):
- New table source PK + request_delay_sec numeric(5,2) + updated_at + description
- Seeded with row 'yandex' = 5.0s (umbrella key for all 4 Yandex scrapers)
scraper_settings.py (new):
- get_scraper_delay(source) -> float, with 60s in-process cache + DB fallback
- Yandex sub-scrapers (yandex_detail / yandex_newbuilding / yandex_valuation)
resolve to umbrella 'yandex' key via _KEY_ALIASES
- invalidate_cache(source | None) for admin PUT side-effect
- Deferred SessionLocal import (_open_session helper) for unit-test compat
4 Yandex scrapers wired to load delay at __init__:
- yandex_realty.py / yandex_detail.py / yandex_newbuilding.py / yandex_valuation.py
- Class default bumped to 5.0; instance value overridden via get_scraper_delay(self.name)
admin.py -- 5 new endpoints:
- GET /scraper-settings list[ScraperSetting]
- PUT /scraper-settings/{source} upsert + invalidate_cache
- POST /scrape/yandex-detail offer_url= -> detail snapshot
- POST /scrape/yandex-newbuilding slug=, id=, city=ekaterinburg -> JK snapshot
- POST /scrape/yandex-valuation address=, offer_category=, offer_type=, page= -> house meta
Tests: 10 + 4 = 14 new unit tests (loader cache/alias/error paths + scraper wiring).
Ruff clean.
101 lines
3 KiB
Python
101 lines
3 KiB
Python
"""Runtime config loader for scraper delays.
|
|
|
|
Backed by scraper_settings table with 60s in-process cache.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
from threading import Lock
|
|
|
|
from sqlalchemy import text
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# In-process cache: {source: (delay_sec, fetched_at_epoch)}
|
|
_CACHE: dict[str, tuple[float, float]] = {}
|
|
_CACHE_LOCK = Lock()
|
|
_CACHE_TTL_SEC = 60.0
|
|
|
|
# Default fallback if DB row missing or DB unreachable
|
|
_DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
|
|
"yandex": 5.0,
|
|
"yandex_detail": 5.0,
|
|
"yandex_newbuilding": 5.0,
|
|
"yandex_valuation": 5.0,
|
|
}
|
|
_GLOBAL_DEFAULT_DELAY = 5.0
|
|
|
|
# Yandex sub-scrapers all read the umbrella "yandex" key.
|
|
# Map: scraper_name -> DB row key.
|
|
_KEY_ALIASES: dict[str, str] = {
|
|
"yandex": "yandex",
|
|
"yandex_detail": "yandex",
|
|
"yandex_newbuilding": "yandex",
|
|
"yandex_realty_nb": "yandex", # BaseScraper.name used by YandexNewbuildingScraper
|
|
"yandex_valuation": "yandex",
|
|
# Non-Yandex sources resolve to their own keys for forward-compat
|
|
# (no rows exist yet for them; fall back to default).
|
|
}
|
|
|
|
|
|
def _open_session() -> object:
|
|
"""Return a new SessionLocal context-manager instance.
|
|
|
|
Deferred import avoids triggering Settings() validation at module import
|
|
time (unit tests run without DATABASE_URL env var).
|
|
Extracted as a named function so tests can patch it.
|
|
"""
|
|
from app.core.db import SessionLocal
|
|
|
|
return SessionLocal()
|
|
|
|
|
|
def get_scraper_delay(source: str) -> float:
|
|
"""Return the configured request_delay_sec for `source`.
|
|
|
|
- Maps Yandex sub-scrapers to the umbrella 'yandex' key.
|
|
- Cached 60s in-process to avoid per-instance DB query.
|
|
- Any DB error -> returns the hardcoded default for that source.
|
|
"""
|
|
key = _KEY_ALIASES.get(source, source)
|
|
|
|
now = time.time()
|
|
with _CACHE_LOCK:
|
|
cached = _CACHE.get(key)
|
|
if cached is not None and now - cached[1] < _CACHE_TTL_SEC:
|
|
return cached[0]
|
|
|
|
try:
|
|
with _open_session() as db:
|
|
row = db.execute(
|
|
text(
|
|
"SELECT request_delay_sec FROM scraper_settings WHERE source = :s"
|
|
),
|
|
{"s": key},
|
|
).first()
|
|
if row is not None:
|
|
value = float(row[0])
|
|
else:
|
|
value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
|
except Exception as e:
|
|
logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
|
|
value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
|
|
|
with _CACHE_LOCK:
|
|
_CACHE[key] = (value, now)
|
|
return value
|
|
|
|
|
|
def invalidate_cache(source: str | None = None) -> None:
|
|
"""Clear the cache entry for `source`, or the whole cache if None.
|
|
|
|
Called by admin endpoint after PUT to force a re-read on next get_scraper_delay().
|
|
"""
|
|
with _CACHE_LOCK:
|
|
if source is None:
|
|
_CACHE.clear()
|
|
else:
|
|
key = _KEY_ALIASES.get(source, source)
|
|
_CACHE.pop(key, None)
|