feat(tradein): global scraper delay setting (applies across all scrapers) #485
7 changed files with 399 additions and 144 deletions
|
|
@ -79,17 +79,6 @@ class ScrapeResponse(BaseModel):
|
||||||
by_source: list[ScrapeResult]
|
by_source: list[ScrapeResult]
|
||||||
|
|
||||||
|
|
||||||
class ScraperSetting(BaseModel):
|
|
||||||
source: str
|
|
||||||
request_delay_sec: float
|
|
||||||
description: str | None = None
|
|
||||||
updated_at: str | None = None # ISO
|
|
||||||
|
|
||||||
|
|
||||||
class ScraperSettingUpdate(BaseModel):
|
|
||||||
request_delay_sec: float = Field(ge=1.0, le=60.0)
|
|
||||||
|
|
||||||
|
|
||||||
@router.post("/scrape", response_model=ScrapeResponse)
|
@router.post("/scrape", response_model=ScrapeResponse)
|
||||||
async def scrape_around(
|
async def scrape_around(
|
||||||
payload: ScrapeRequest,
|
payload: ScrapeRequest,
|
||||||
|
|
@ -560,6 +549,74 @@ def cancel_avito_city_sweep(
|
||||||
return {"ok": True, "run_id": run_id, "cancelled": cancelled}
|
return {"ok": True, "run_id": run_id, "cancelled": cancelled}
|
||||||
|
|
||||||
|
|
||||||
|
# ── Scraper settings: global + per-source delay management ───────────────────
|
||||||
|
|
||||||
|
|
||||||
|
class ScraperSettingPayload(BaseModel):
|
||||||
|
source: str = Field(..., min_length=1, max_length=64)
|
||||||
|
request_delay_sec: float = Field(..., ge=0.0, le=60.0)
|
||||||
|
description: str | None = None
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/scraper-settings", status_code=200)
|
||||||
|
def list_scraper_settings(
|
||||||
|
db: Annotated[Session, Depends(get_db)],
|
||||||
|
) -> dict:
|
||||||
|
"""Список всех настроек задержки (per-source + global)."""
|
||||||
|
rows = db.execute(
|
||||||
|
text("""
|
||||||
|
SELECT source, request_delay_sec, description, updated_at
|
||||||
|
FROM scraper_settings
|
||||||
|
ORDER BY source ASC
|
||||||
|
""")
|
||||||
|
).mappings().all()
|
||||||
|
return {"settings": [dict(r) for r in rows]}
|
||||||
|
|
||||||
|
|
||||||
|
@router.put("/scraper-settings/{source}", status_code=200)
|
||||||
|
def update_scraper_setting(
|
||||||
|
source: str,
|
||||||
|
payload: ScraperSettingPayload,
|
||||||
|
db: Annotated[Session, Depends(get_db)],
|
||||||
|
) -> dict:
|
||||||
|
"""Обновить задержку для source (или 'global'). Path source используется как ключ.
|
||||||
|
|
||||||
|
Эффект применяется немедленно — invalidate_cache() сбрасывает кеш для source,
|
||||||
|
следующий get_scraper_delay() перечитает из БД.
|
||||||
|
|
||||||
|
source='global' — нижняя планка для всех парсеров (0 = отключено).
|
||||||
|
"""
|
||||||
|
if payload.source != source:
|
||||||
|
raise HTTPException(
|
||||||
|
status_code=400,
|
||||||
|
detail=f"Path source={source!r} does not match body source={payload.source!r}",
|
||||||
|
)
|
||||||
|
|
||||||
|
row = db.execute(
|
||||||
|
text("""
|
||||||
|
INSERT INTO scraper_settings (source, request_delay_sec, description, updated_at)
|
||||||
|
VALUES (:s, CAST(:d AS numeric), :desc, NOW())
|
||||||
|
ON CONFLICT (source) DO UPDATE SET
|
||||||
|
request_delay_sec = EXCLUDED.request_delay_sec,
|
||||||
|
description = COALESCE(EXCLUDED.description, scraper_settings.description),
|
||||||
|
updated_at = NOW()
|
||||||
|
RETURNING source, request_delay_sec, description, updated_at
|
||||||
|
"""),
|
||||||
|
{
|
||||||
|
"s": source,
|
||||||
|
"d": payload.request_delay_sec,
|
||||||
|
"desc": payload.description,
|
||||||
|
},
|
||||||
|
).mappings().one()
|
||||||
|
db.commit()
|
||||||
|
|
||||||
|
invalidate_cache(source)
|
||||||
|
logger.info(
|
||||||
|
"scraper-settings: updated source=%s delay=%.1f", source, payload.request_delay_sec
|
||||||
|
)
|
||||||
|
return dict(row)
|
||||||
|
|
||||||
|
|
||||||
# ── In-app scheduler endpoints (Stage 4e) ────────────────────────────────────
|
# ── In-app scheduler endpoints (Stage 4e) ────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -649,78 +706,6 @@ def update_schedule(
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
# -- Scraper settings (per-source live config) --------------------------------
|
|
||||||
|
|
||||||
|
|
||||||
@router.get("/scraper-settings", response_model=list[ScraperSetting])
|
|
||||||
async def list_scraper_settings(
|
|
||||||
db: Annotated[Session, Depends(get_db)],
|
|
||||||
) -> list[ScraperSetting]:
|
|
||||||
"""Return all scraper_settings rows (live config for delays)."""
|
|
||||||
rows = (
|
|
||||||
db.execute(
|
|
||||||
text(
|
|
||||||
"""
|
|
||||||
SELECT source, request_delay_sec, description, updated_at
|
|
||||||
FROM scraper_settings
|
|
||||||
ORDER BY source
|
|
||||||
"""
|
|
||||||
)
|
|
||||||
)
|
|
||||||
.mappings()
|
|
||||||
.all()
|
|
||||||
)
|
|
||||||
return [
|
|
||||||
ScraperSetting(
|
|
||||||
source=r["source"],
|
|
||||||
request_delay_sec=float(r["request_delay_sec"]),
|
|
||||||
description=r["description"],
|
|
||||||
updated_at=r["updated_at"].isoformat() if r["updated_at"] else None,
|
|
||||||
)
|
|
||||||
for r in rows
|
|
||||||
]
|
|
||||||
|
|
||||||
|
|
||||||
@router.put("/scraper-settings/{source}", response_model=ScraperSetting)
|
|
||||||
async def update_scraper_setting(
|
|
||||||
source: str,
|
|
||||||
payload: ScraperSettingUpdate,
|
|
||||||
db: Annotated[Session, Depends(get_db)],
|
|
||||||
) -> ScraperSetting:
|
|
||||||
"""Update request_delay_sec for one source. UPSERT - creates row if missing.
|
|
||||||
|
|
||||||
Invalidates the in-process cache so the change takes effect on next
|
|
||||||
scraper instantiation (usually within seconds).
|
|
||||||
"""
|
|
||||||
row = (
|
|
||||||
db.execute(
|
|
||||||
text(
|
|
||||||
"""
|
|
||||||
INSERT INTO scraper_settings (source, request_delay_sec, updated_at)
|
|
||||||
VALUES (:source, :delay, NOW())
|
|
||||||
ON CONFLICT (source) DO UPDATE
|
|
||||||
SET request_delay_sec = EXCLUDED.request_delay_sec,
|
|
||||||
updated_at = NOW()
|
|
||||||
RETURNING source, request_delay_sec, description, updated_at
|
|
||||||
"""
|
|
||||||
),
|
|
||||||
{"source": source, "delay": payload.request_delay_sec},
|
|
||||||
)
|
|
||||||
.mappings()
|
|
||||||
.first()
|
|
||||||
)
|
|
||||||
db.commit()
|
|
||||||
invalidate_cache(source)
|
|
||||||
if row is None:
|
|
||||||
raise HTTPException(500, "Failed to upsert scraper_settings row")
|
|
||||||
return ScraperSetting(
|
|
||||||
source=row["source"],
|
|
||||||
request_delay_sec=float(row["request_delay_sec"]),
|
|
||||||
description=row["description"],
|
|
||||||
updated_at=row["updated_at"].isoformat() if row["updated_at"] else None,
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# -- Yandex ad-hoc scrape triggers --------------------------------------------
|
# -- Yandex ad-hoc scrape triggers --------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,12 @@
|
||||||
"""Runtime config loader for scraper delays.
|
"""scraper_settings.py — загрузка задержек парсеров из таблицы scraper_settings.
|
||||||
|
|
||||||
Backed by scraper_settings table with 60s in-process cache.
|
Кеш в памяти с TTL 60 секунд — не нагружает БД при каждом запросе.
|
||||||
|
invalidate_cache() вызывается из admin API PUT для немедленного применения.
|
||||||
|
|
||||||
|
Ключевая логика:
|
||||||
|
get_scraper_delay(source) = max(per_source_delay, global_delay)
|
||||||
|
Строка source='global' задаёт нижнюю планку для ВСЕХ парсеров.
|
||||||
|
Если global=0 — используется только per-source значение.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
@ -13,6 +19,9 @@ from sqlalchemy import text
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
# Специальный ключ глобальной задержки (строка в scraper_settings с этим source).
|
||||||
|
_GLOBAL_KEY = "global"
|
||||||
|
|
||||||
# In-process cache: {source: (delay_sec, fetched_at_epoch)}
|
# In-process cache: {source: (delay_sec, fetched_at_epoch)}
|
||||||
_CACHE: dict[str, tuple[float, float]] = {}
|
_CACHE: dict[str, tuple[float, float]] = {}
|
||||||
_CACHE_LOCK = Lock()
|
_CACHE_LOCK = Lock()
|
||||||
|
|
@ -20,10 +29,15 @@ _CACHE_TTL_SEC = 60.0
|
||||||
|
|
||||||
# Default fallback if DB row missing or DB unreachable
|
# Default fallback if DB row missing or DB unreachable
|
||||||
_DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
|
_DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
|
||||||
|
"avito": 7.0,
|
||||||
|
"cian": 5.0,
|
||||||
|
"n1": 5.0,
|
||||||
"yandex": 5.0,
|
"yandex": 5.0,
|
||||||
"yandex_detail": 5.0,
|
"yandex_detail": 5.0,
|
||||||
"yandex_newbuilding": 5.0,
|
"yandex_newbuilding": 5.0,
|
||||||
"yandex_valuation": 5.0,
|
"yandex_valuation": 5.0,
|
||||||
|
"domrf": 5.0,
|
||||||
|
"rosreestr": 5.0,
|
||||||
}
|
}
|
||||||
_GLOBAL_DEFAULT_DELAY = 5.0
|
_GLOBAL_DEFAULT_DELAY = 5.0
|
||||||
|
|
||||||
|
|
@ -52,15 +66,13 @@ def _open_session() -> object:
|
||||||
return SessionLocal()
|
return SessionLocal()
|
||||||
|
|
||||||
|
|
||||||
def get_scraper_delay(source: str) -> float:
|
def _get_setting_cached(key: str) -> float:
|
||||||
"""Return the configured request_delay_sec for `source`.
|
"""Cache-aware read of one source key from scraper_settings.
|
||||||
|
|
||||||
- Maps Yandex sub-scrapers to the umbrella 'yandex' key.
|
Returns request_delay_sec from DB (with TTL=60s cache).
|
||||||
- Cached 60s in-process to avoid per-instance DB query.
|
Missing row: falls back to _DEFAULT_DELAY_BY_SOURCE, or 0.0 for _GLOBAL_KEY
|
||||||
- Any DB error -> returns the hardcoded default for that source.
|
(meaning no global floor is applied when the row hasn't been seeded yet).
|
||||||
"""
|
"""
|
||||||
key = _KEY_ALIASES.get(source, source)
|
|
||||||
|
|
||||||
now = time.time()
|
now = time.time()
|
||||||
with _CACHE_LOCK:
|
with _CACHE_LOCK:
|
||||||
cached = _CACHE.get(key)
|
cached = _CACHE.get(key)
|
||||||
|
|
@ -77,17 +89,37 @@ def get_scraper_delay(source: str) -> float:
|
||||||
).first()
|
).first()
|
||||||
if row is not None:
|
if row is not None:
|
||||||
value = float(row[0])
|
value = float(row[0])
|
||||||
|
elif key == _GLOBAL_KEY:
|
||||||
|
# Global row not yet seeded — don't apply floor.
|
||||||
|
value = 0.0
|
||||||
else:
|
else:
|
||||||
value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
|
logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
|
||||||
value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(
|
||||||
|
key, _GLOBAL_DEFAULT_DELAY
|
||||||
|
)
|
||||||
|
|
||||||
with _CACHE_LOCK:
|
with _CACHE_LOCK:
|
||||||
_CACHE[key] = (value, now)
|
_CACHE[key] = (value, now)
|
||||||
return value
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def get_scraper_delay(source: str) -> float:
|
||||||
|
"""Return the configured request_delay_sec for `source`.
|
||||||
|
|
||||||
|
Effective delay = max(per_source, global).
|
||||||
|
- Maps Yandex sub-scrapers to the umbrella 'yandex' key.
|
||||||
|
- Cached 60s in-process to avoid per-instance DB query.
|
||||||
|
- source='global' sets a floor across all scrapers (0 = disabled).
|
||||||
|
- Any DB error -> returns the hardcoded default for that source.
|
||||||
|
"""
|
||||||
|
key = _KEY_ALIASES.get(source, source)
|
||||||
|
per_source = _get_setting_cached(key)
|
||||||
|
global_value = _get_setting_cached(_GLOBAL_KEY)
|
||||||
|
return max(per_source, global_value)
|
||||||
|
|
||||||
|
|
||||||
def invalidate_cache(source: str | None = None) -> None:
|
def invalidate_cache(source: str | None = None) -> None:
|
||||||
"""Clear the cache entry for `source`, or the whole cache if None.
|
"""Clear the cache entry for `source`, or the whole cache if None.
|
||||||
|
|
||||||
|
|
@ -96,6 +128,11 @@ def invalidate_cache(source: str | None = None) -> None:
|
||||||
with _CACHE_LOCK:
|
with _CACHE_LOCK:
|
||||||
if source is None:
|
if source is None:
|
||||||
_CACHE.clear()
|
_CACHE.clear()
|
||||||
|
logger.info("scraper_settings: cache cleared (all sources)")
|
||||||
else:
|
else:
|
||||||
key = _KEY_ALIASES.get(source, source)
|
key = _KEY_ALIASES.get(source, source)
|
||||||
_CACHE.pop(key, None)
|
_CACHE.pop(key, None)
|
||||||
|
# Also clear the source itself if it differed from key (alias case).
|
||||||
|
if key != source:
|
||||||
|
_CACHE.pop(source, None)
|
||||||
|
logger.info("scraper_settings: cache cleared for source=%s", source)
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,7 @@ from urllib.parse import urlencode, urljoin
|
||||||
from curl_cffi.requests import AsyncSession
|
from curl_cffi.requests import AsyncSession
|
||||||
from selectolax.parser import HTMLParser
|
from selectolax.parser import HTMLParser
|
||||||
|
|
||||||
|
from app.services.scraper_settings import get_scraper_delay
|
||||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -36,11 +37,13 @@ class AvitoScraper(BaseScraper):
|
||||||
|
|
||||||
name = "avito"
|
name = "avito"
|
||||||
base_url = "https://www.avito.ru"
|
base_url = "https://www.avito.ru"
|
||||||
# Avito жёстко мониторит — спим долго между запросами
|
# Avito жёстко мониторит — спим долго между запросами.
|
||||||
|
# Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра.
|
||||||
request_delay_sec = 7.0
|
request_delay_sec = 7.0
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
self.request_delay_sec = get_scraper_delay(self.name)
|
||||||
self._cffi: AsyncSession | None = None
|
self._cffi: AsyncSession | None = None
|
||||||
|
|
||||||
async def __aenter__(self) -> AvitoScraper:
|
async def __aenter__(self) -> AvitoScraper:
|
||||||
|
|
|
||||||
|
|
@ -23,6 +23,7 @@ from urllib.parse import urlencode
|
||||||
|
|
||||||
from curl_cffi.requests import AsyncSession
|
from curl_cffi.requests import AsyncSession
|
||||||
|
|
||||||
|
from app.services.scraper_settings import get_scraper_delay
|
||||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||||
from app.services.scrapers.cian_state_parser import extract_state
|
from app.services.scrapers.cian_state_parser import extract_state
|
||||||
|
|
||||||
|
|
@ -46,10 +47,12 @@ class CianScraper(BaseScraper):
|
||||||
name = "cian"
|
name = "cian"
|
||||||
# ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema sec 13, closed Q4)
|
# ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema sec 13, closed Q4)
|
||||||
base_url = "https://ekb.cian.ru"
|
base_url = "https://ekb.cian.ru"
|
||||||
|
# Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра.
|
||||||
request_delay_sec = 5.0 # консервативно: Cian менее агрессивен чем Avito, но 5s безопасно
|
request_delay_sec = 5.0 # консервативно: Cian менее агрессивен чем Avito, но 5s безопасно
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
self.request_delay_sec = get_scraper_delay(self.name)
|
||||||
self._cffi: AsyncSession | None = None
|
self._cffi: AsyncSession | None = None
|
||||||
|
|
||||||
async def __aenter__(self) -> CianScraper:
|
async def __aenter__(self) -> CianScraper:
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,7 @@ from typing import Any
|
||||||
from curl_cffi.requests import AsyncSession
|
from curl_cffi.requests import AsyncSession
|
||||||
from selectolax.parser import HTMLParser
|
from selectolax.parser import HTMLParser
|
||||||
|
|
||||||
|
from app.services.scraper_settings import get_scraper_delay
|
||||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
|
|
@ -35,13 +36,15 @@ class N1Scraper(BaseScraper):
|
||||||
|
|
||||||
name = "n1"
|
name = "n1"
|
||||||
base_url = "https://ekaterinburg.n1.ru"
|
base_url = "https://ekaterinburg.n1.ru"
|
||||||
|
# Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра.
|
||||||
request_delay_sec = 5.0
|
request_delay_sec = 5.0
|
||||||
|
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
super().__init__()
|
super().__init__()
|
||||||
|
self.request_delay_sec = get_scraper_delay(self.name)
|
||||||
self._cffi: AsyncSession | None = None
|
self._cffi: AsyncSession | None = None
|
||||||
|
|
||||||
async def __aenter__(self) -> "N1Scraper":
|
async def __aenter__(self) -> N1Scraper:
|
||||||
await super().__aenter__()
|
await super().__aenter__()
|
||||||
self._cffi = AsyncSession(
|
self._cffi = AsyncSession(
|
||||||
impersonate="chrome120",
|
impersonate="chrome120",
|
||||||
|
|
@ -74,7 +77,7 @@ class N1Scraper(BaseScraper):
|
||||||
try:
|
try:
|
||||||
assert self._cffi is not None
|
assert self._cffi is not None
|
||||||
response = await self._cffi.get(url)
|
response = await self._cffi.get(url)
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
logger.exception("n1 fetch failed for %s", url)
|
logger.exception("n1 fetch failed for %s", url)
|
||||||
return []
|
return []
|
||||||
if response.status_code != 200:
|
if response.status_code != 200:
|
||||||
|
|
@ -153,7 +156,9 @@ class N1Scraper(BaseScraper):
|
||||||
title = link.text(strip=True) or ""
|
title = link.text(strip=True) or ""
|
||||||
rooms = _extract_rooms(title)
|
rooms = _extract_rooms(title)
|
||||||
# Адрес = title без префикса комнат
|
# Адрес = title без префикса комнат
|
||||||
address = re.sub(r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE).strip()
|
address = re.sub(
|
||||||
|
r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE
|
||||||
|
).strip()
|
||||||
if not address:
|
if not address:
|
||||||
address = "Екатеринбург (N1)"
|
address = "Екатеринбург (N1)"
|
||||||
|
|
||||||
|
|
@ -192,7 +197,7 @@ class N1Scraper(BaseScraper):
|
||||||
photo_urls=photos,
|
photo_urls=photos,
|
||||||
raw_payload={"title": title},
|
raw_payload={"title": title},
|
||||||
)
|
)
|
||||||
except Exception: # noqa: BLE001
|
except Exception:
|
||||||
logger.exception("n1 card parse failed")
|
logger.exception("n1 card parse failed")
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
|
||||||
25
tradein-mvp/backend/data/sql/054_scraper_settings_global.sql
Normal file
25
tradein-mvp/backend/data/sql/054_scraper_settings_global.sql
Normal file
|
|
@ -0,0 +1,25 @@
|
||||||
|
-- 054_scraper_settings_global.sql
|
||||||
|
-- Add global delay row + per-source aliases for all scrapers.
|
||||||
|
-- Idempotent — ON CONFLICT DO NOTHING preserves user-edited values.
|
||||||
|
BEGIN;
|
||||||
|
|
||||||
|
-- 'global' row: when > 0, acts as floor (max with per-source delay).
|
||||||
|
-- Default 0 = use per-source defaults.
|
||||||
|
INSERT INTO scraper_settings (source, request_delay_sec, description)
|
||||||
|
VALUES (
|
||||||
|
'global',
|
||||||
|
0.0,
|
||||||
|
'Global delay applied across ALL scrapers (max with per-source). 0 = use per-source defaults.'
|
||||||
|
)
|
||||||
|
ON CONFLICT (source) DO NOTHING;
|
||||||
|
|
||||||
|
-- Per-source rows for all scrapers (individual control).
|
||||||
|
INSERT INTO scraper_settings (source, request_delay_sec, description) VALUES
|
||||||
|
('avito', 7.0, 'Avito SERP delay (anti-ban)'),
|
||||||
|
('cian', 5.0, 'Cian SERP delay'),
|
||||||
|
('n1', 5.0, 'N1 SERP delay'),
|
||||||
|
('domrf', 5.0, 'DOM.RF SERP delay'),
|
||||||
|
('rosreestr', 5.0, 'Rosreestr API delay')
|
||||||
|
ON CONFLICT (source) DO NOTHING;
|
||||||
|
|
||||||
|
COMMIT;
|
||||||
|
|
@ -1,14 +1,30 @@
|
||||||
"""Tests for scraper_settings loader (cache + DB fallback + alias mapping)."""
|
"""Tests for scraper_settings loader (cache + DB fallback + alias mapping + global delay).
|
||||||
|
|
||||||
|
Covers:
|
||||||
|
- Per-source value lookup + caching (main's tests)
|
||||||
|
- Global delay max() logic (our additions)
|
||||||
|
- Admin endpoint smoke
|
||||||
|
"""
|
||||||
|
|
||||||
|
import os
|
||||||
|
|
||||||
|
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||||
|
|
||||||
from unittest.mock import MagicMock, patch
|
from unittest.mock import MagicMock, patch
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
|
import app.services.scraper_settings as ss_mod
|
||||||
from app.services import scraper_settings as ss
|
from app.services import scraper_settings as ss
|
||||||
|
from app.services.scraper_settings import get_scraper_delay, invalidate_cache
|
||||||
|
|
||||||
# Patch target: _open_session is a module-level wrapper so tests can mock DB
|
# Patch target: _open_session is a module-level wrapper so tests can mock DB
|
||||||
_PATCH_TARGET = "app.services.scraper_settings._open_session"
|
_PATCH_TARGET = "app.services.scraper_settings._open_session"
|
||||||
|
|
||||||
|
|
||||||
|
# ── Fixtures ──────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture(autouse=True)
|
@pytest.fixture(autouse=True)
|
||||||
def _reset_cache():
|
def _reset_cache():
|
||||||
ss._CACHE.clear()
|
ss._CACHE.clear()
|
||||||
|
|
@ -16,8 +32,15 @@ def _reset_cache():
|
||||||
ss._CACHE.clear()
|
ss._CACHE.clear()
|
||||||
|
|
||||||
|
|
||||||
def _mock_session(row_value: float | None) -> MagicMock:
|
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||||
"""Return a mock that behaves like SessionLocal() context-manager."""
|
|
||||||
|
|
||||||
|
def _mock_session_uniform(row_value: float | None) -> tuple[MagicMock, MagicMock]:
|
||||||
|
"""Mock that returns the same value for any key (context-manager style).
|
||||||
|
|
||||||
|
Used by tests that don't care about multi-key distinction.
|
||||||
|
Global key will also return the same value, so max() returns row_value.
|
||||||
|
"""
|
||||||
session = MagicMock()
|
session = MagicMock()
|
||||||
session.__enter__ = MagicMock(return_value=session)
|
session.__enter__ = MagicMock(return_value=session)
|
||||||
session.__exit__ = MagicMock(return_value=False)
|
session.__exit__ = MagicMock(return_value=False)
|
||||||
|
|
@ -25,56 +48,86 @@ def _mock_session(row_value: float | None) -> MagicMock:
|
||||||
session.execute.return_value.first.return_value = None
|
session.execute.return_value.first.return_value = None
|
||||||
else:
|
else:
|
||||||
session.execute.return_value.first.return_value = (row_value,)
|
session.execute.return_value.first.return_value = (row_value,)
|
||||||
# _open_session() returns the session directly (no extra call)
|
|
||||||
mock_open = MagicMock(return_value=session)
|
mock_open = MagicMock(return_value=session)
|
||||||
return mock_open, session
|
return mock_open, session
|
||||||
|
|
||||||
|
|
||||||
|
def _make_mock_session(values: dict[str, float | None]) -> MagicMock:
|
||||||
|
"""Mock сессии БД с фиксированными значениями per source key.
|
||||||
|
|
||||||
|
Поддерживает разные значения для разных ключей — используется в тестах
|
||||||
|
global delay логики.
|
||||||
|
"""
|
||||||
|
mock_db = MagicMock()
|
||||||
|
|
||||||
|
def fake_execute(sql, params):
|
||||||
|
key = params["s"]
|
||||||
|
result = MagicMock()
|
||||||
|
val = values.get(key)
|
||||||
|
result.first.return_value = (val,) if val is not None else None
|
||||||
|
return result
|
||||||
|
|
||||||
|
mock_db.execute = fake_execute
|
||||||
|
mock_db.__enter__ = MagicMock(return_value=mock_db)
|
||||||
|
mock_db.__exit__ = MagicMock(return_value=False)
|
||||||
|
return mock_db
|
||||||
|
|
||||||
|
|
||||||
|
# ── Main's tests: core cache + alias + fallback behaviour ─────────────────────
|
||||||
|
|
||||||
|
|
||||||
def test_returns_db_value_for_yandex():
|
def test_returns_db_value_for_yandex():
|
||||||
mock_open, _ = _mock_session(7.5)
|
"""DB value 7.5, global also 7.5 → max(7.5, 7.5) = 7.5."""
|
||||||
|
mock_open, _ = _mock_session_uniform(7.5)
|
||||||
with patch(_PATCH_TARGET, mock_open):
|
with patch(_PATCH_TARGET, mock_open):
|
||||||
assert ss.get_scraper_delay("yandex") == 7.5
|
assert ss.get_scraper_delay("yandex") == 7.5
|
||||||
|
|
||||||
|
|
||||||
def test_yandex_subkeys_map_to_yandex_umbrella():
|
def test_yandex_subkeys_map_to_yandex_umbrella():
|
||||||
"""yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row."""
|
"""yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row."""
|
||||||
mock_open, _session = _mock_session(8.0)
|
mock_open, _session = _mock_session_uniform(8.0)
|
||||||
with patch(_PATCH_TARGET, mock_open):
|
with patch(_PATCH_TARGET, mock_open):
|
||||||
assert ss.get_scraper_delay("yandex_detail") == 8.0
|
assert ss.get_scraper_delay("yandex_detail") == 8.0
|
||||||
assert ss.get_scraper_delay("yandex_newbuilding") == 8.0
|
assert ss.get_scraper_delay("yandex_newbuilding") == 8.0
|
||||||
assert ss.get_scraper_delay("yandex_valuation") == 8.0
|
assert ss.get_scraper_delay("yandex_valuation") == 8.0
|
||||||
# All 3 lookups hit the cache after the first; _open_session called once
|
# First call fetches yandex + global (2 DB calls). Remaining 2 subkey calls
|
||||||
assert mock_open.call_count == 1
|
# map to "yandex" (already cached) and "global" (already cached) → 0 more.
|
||||||
|
|
||||||
|
|
||||||
def test_cache_hit_within_ttl():
|
|
||||||
mock_open, _session = _mock_session(6.0)
|
|
||||||
with patch(_PATCH_TARGET, mock_open):
|
|
||||||
ss.get_scraper_delay("yandex")
|
|
||||||
ss.get_scraper_delay("yandex")
|
|
||||||
ss.get_scraper_delay("yandex")
|
|
||||||
assert mock_open.call_count == 1 # cached after first
|
|
||||||
|
|
||||||
|
|
||||||
def test_cache_expires_after_ttl():
|
|
||||||
mock_open, _session = _mock_session(6.0)
|
|
||||||
with patch(_PATCH_TARGET, mock_open):
|
|
||||||
ss.get_scraper_delay("yandex")
|
|
||||||
# Force-age the cache
|
|
||||||
with ss._CACHE_LOCK:
|
|
||||||
v, t = ss._CACHE["yandex"]
|
|
||||||
ss._CACHE["yandex"] = (v, t - 999)
|
|
||||||
ss.get_scraper_delay("yandex")
|
|
||||||
assert mock_open.call_count == 2
|
assert mock_open.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_hit_within_ttl():
|
||||||
|
"""After first call both yandex+global are cached; repeats hit cache only."""
|
||||||
|
mock_open, _session = _mock_session_uniform(6.0)
|
||||||
|
with patch(_PATCH_TARGET, mock_open):
|
||||||
|
ss.get_scraper_delay("yandex")
|
||||||
|
ss.get_scraper_delay("yandex")
|
||||||
|
ss.get_scraper_delay("yandex")
|
||||||
|
# First call: 2 DB hits (yandex + global). Subsequent: 0.
|
||||||
|
assert mock_open.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_cache_expires_after_ttl():
|
||||||
|
"""Force-age yandex key — next call re-fetches yandex only (global still fresh)."""
|
||||||
|
mock_open, _session = _mock_session_uniform(6.0)
|
||||||
|
with patch(_PATCH_TARGET, mock_open):
|
||||||
|
ss.get_scraper_delay("yandex") # +2 calls (yandex + global)
|
||||||
|
# Force-age the yandex entry
|
||||||
|
with ss._CACHE_LOCK:
|
||||||
|
v, t = ss._CACHE["yandex"]
|
||||||
|
ss._CACHE["yandex"] = (v, t - 999)
|
||||||
|
ss.get_scraper_delay("yandex") # +1 call (yandex stale; global still fresh)
|
||||||
|
assert mock_open.call_count == 3
|
||||||
|
|
||||||
|
|
||||||
def test_db_missing_row_returns_default():
|
def test_db_missing_row_returns_default():
|
||||||
mock_open, _ = _mock_session(None)
|
"""Missing yandex row → _DEFAULT_DELAY_BY_SOURCE. Global None → 0.0. max = default."""
|
||||||
|
mock_open, _ = _mock_session_uniform(None)
|
||||||
with patch(_PATCH_TARGET, mock_open):
|
with patch(_PATCH_TARGET, mock_open):
|
||||||
assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]
|
assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]
|
||||||
|
|
||||||
|
|
||||||
def test_db_error_returns_default():
|
def test_db_error_returns_default():
|
||||||
|
"""DB error → hardcoded default. Global error → 0.0. max = default."""
|
||||||
bad_session = MagicMock()
|
bad_session = MagicMock()
|
||||||
bad_session.__enter__ = MagicMock(return_value=bad_session)
|
bad_session.__enter__ = MagicMock(return_value=bad_session)
|
||||||
bad_session.__exit__ = MagicMock(return_value=False)
|
bad_session.__exit__ = MagicMock(return_value=False)
|
||||||
|
|
@ -85,34 +138,178 @@ def test_db_error_returns_default():
|
||||||
|
|
||||||
|
|
||||||
def test_unknown_source_uses_global_default():
|
def test_unknown_source_uses_global_default():
|
||||||
mock_open, _ = _mock_session(None)
|
"""Unknown source, DB row None → _GLOBAL_DEFAULT_DELAY. Global None → 0.0. max = 5.0."""
|
||||||
|
mock_open, _ = _mock_session_uniform(None)
|
||||||
with patch(_PATCH_TARGET, mock_open):
|
with patch(_PATCH_TARGET, mock_open):
|
||||||
assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY
|
assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY
|
||||||
|
|
||||||
|
|
||||||
def test_invalidate_cache_specific_source():
|
def test_invalidate_cache_specific_source():
|
||||||
mock_open, _ = _mock_session(6.0)
|
"""invalidate_cache(yandex) drops yandex entry; next call re-fetches yandex only."""
|
||||||
|
mock_open, _ = _mock_session_uniform(6.0)
|
||||||
with patch(_PATCH_TARGET, mock_open):
|
with patch(_PATCH_TARGET, mock_open):
|
||||||
ss.get_scraper_delay("yandex")
|
ss.get_scraper_delay("yandex") # +2 (yandex + global)
|
||||||
ss.invalidate_cache("yandex")
|
ss.invalidate_cache("yandex") # clears yandex key
|
||||||
ss.get_scraper_delay("yandex")
|
ss.get_scraper_delay("yandex") # +1 (yandex stale; global still cached)
|
||||||
assert mock_open.call_count == 2
|
assert mock_open.call_count == 3
|
||||||
|
|
||||||
|
|
||||||
def test_invalidate_cache_all():
|
def test_invalidate_cache_all():
|
||||||
mock_open, _ = _mock_session(6.0)
|
"""invalidate_cache() clears all entries; next call re-fetches both."""
|
||||||
|
mock_open, _ = _mock_session_uniform(6.0)
|
||||||
with patch(_PATCH_TARGET, mock_open):
|
with patch(_PATCH_TARGET, mock_open):
|
||||||
ss.get_scraper_delay("yandex")
|
ss.get_scraper_delay("yandex") # +2 (yandex + global)
|
||||||
ss.invalidate_cache()
|
ss.invalidate_cache() # clears everything
|
||||||
ss.get_scraper_delay("yandex")
|
ss.get_scraper_delay("yandex") # +2 (both stale)
|
||||||
assert mock_open.call_count == 2
|
assert mock_open.call_count == 4
|
||||||
|
|
||||||
|
|
||||||
def test_invalidate_alias_invalidates_umbrella():
|
def test_invalidate_alias_invalidates_umbrella():
|
||||||
"""invalidate_cache('yandex_detail') should clear the 'yandex' umbrella key."""
|
"""invalidate_cache('yandex_detail') clears the 'yandex' umbrella key."""
|
||||||
mock_open, _ = _mock_session(6.0)
|
mock_open, _ = _mock_session_uniform(6.0)
|
||||||
with patch(_PATCH_TARGET, mock_open):
|
with patch(_PATCH_TARGET, mock_open):
|
||||||
ss.get_scraper_delay("yandex_detail")
|
ss.get_scraper_delay("yandex_detail") # +2 (yandex + global)
|
||||||
ss.invalidate_cache("yandex_detail")
|
ss.invalidate_cache("yandex_detail") # clears yandex (via alias)
|
||||||
ss.get_scraper_delay("yandex_detail")
|
ss.get_scraper_delay("yandex_detail") # +1 (yandex stale; global fresh)
|
||||||
assert mock_open.call_count == 2
|
assert mock_open.call_count == 3
|
||||||
|
|
||||||
|
|
||||||
|
# ── Our additions: global delay max() logic ───────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_global_delay_takes_precedence_when_higher(monkeypatch):
|
||||||
|
"""global=10 > avito=7 → returns 10."""
|
||||||
|
invalidate_cache()
|
||||||
|
|
||||||
|
mock_db = _make_mock_session({"avito": 7.0, "global": 10.0})
|
||||||
|
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||||
|
|
||||||
|
delay = get_scraper_delay("avito")
|
||||||
|
assert delay == 10.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_per_source_takes_precedence_when_higher(monkeypatch):
|
||||||
|
"""avito=7 > global=2 → returns 7."""
|
||||||
|
invalidate_cache()
|
||||||
|
|
||||||
|
mock_db = _make_mock_session({"avito": 7.0, "global": 2.0})
|
||||||
|
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||||
|
|
||||||
|
delay = get_scraper_delay("avito")
|
||||||
|
assert delay == 7.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_global_zero_means_no_floor(monkeypatch):
|
||||||
|
"""global=0 → only per-source value applies."""
|
||||||
|
invalidate_cache()
|
||||||
|
|
||||||
|
mock_db = _make_mock_session({"cian": 5.0, "global": 0.0})
|
||||||
|
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||||
|
|
||||||
|
delay = get_scraper_delay("cian")
|
||||||
|
assert delay == 5.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_per_source_row_uses_class_default(monkeypatch):
|
||||||
|
"""Missing source row → _DEFAULT_DELAY_BY_SOURCE. global=0 → no floor."""
|
||||||
|
invalidate_cache()
|
||||||
|
|
||||||
|
mock_db = _make_mock_session({"n1": None, "global": 0.0})
|
||||||
|
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||||
|
|
||||||
|
delay = get_scraper_delay("n1")
|
||||||
|
assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["n1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_global_missing_row_treated_as_zero(monkeypatch):
|
||||||
|
"""Missing 'global' row → fallback 0.0, avito=7.0 → max(7.0, 0.0) = 7.0."""
|
||||||
|
invalidate_cache()
|
||||||
|
|
||||||
|
mock_db = _make_mock_session({"avito": 7.0, "global": None})
|
||||||
|
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||||
|
|
||||||
|
delay = get_scraper_delay("avito")
|
||||||
|
assert delay == 7.0
|
||||||
|
|
||||||
|
|
||||||
|
def test_db_error_falls_back_to_default_global_logic(monkeypatch):
|
||||||
|
"""DB error → graceful fallback, no crash. avito default=7.0, global error→0.0."""
|
||||||
|
invalidate_cache()
|
||||||
|
|
||||||
|
def broken_open():
|
||||||
|
raise RuntimeError("DB is down")
|
||||||
|
|
||||||
|
monkeypatch.setattr(ss_mod, "_open_session", broken_open)
|
||||||
|
|
||||||
|
delay = get_scraper_delay("avito")
|
||||||
|
assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["avito"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalidate_cache_single_source_global_logic(monkeypatch):
|
||||||
|
"""invalidate_cache(source) removes only that source from cache."""
|
||||||
|
invalidate_cache()
|
||||||
|
|
||||||
|
mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0})
|
||||||
|
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||||
|
|
||||||
|
get_scraper_delay("cian")
|
||||||
|
get_scraper_delay("avito")
|
||||||
|
|
||||||
|
invalidate_cache("cian")
|
||||||
|
|
||||||
|
with ss_mod._CACHE_LOCK:
|
||||||
|
assert "cian" not in ss_mod._CACHE
|
||||||
|
assert "avito" in ss_mod._CACHE
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalidate_cache_all_global_logic(monkeypatch):
|
||||||
|
"""invalidate_cache() without args clears the entire cache."""
|
||||||
|
invalidate_cache()
|
||||||
|
|
||||||
|
mock_db = _make_mock_session({"avito": 7.0, "global": 0.0})
|
||||||
|
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||||
|
|
||||||
|
get_scraper_delay("avito")
|
||||||
|
invalidate_cache()
|
||||||
|
|
||||||
|
with ss_mod._CACHE_LOCK:
|
||||||
|
assert len(ss_mod._CACHE) == 0
|
||||||
|
|
||||||
|
|
||||||
|
# ── Admin API endpoint smoke ───────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def test_list_scraper_settings_endpoint(monkeypatch):
|
||||||
|
"""GET /scraper-settings mounts and returns a list including 'global'."""
|
||||||
|
import sys
|
||||||
|
|
||||||
|
# Stub weasyprint (not on CI)
|
||||||
|
_wp = MagicMock()
|
||||||
|
sys.modules.setdefault("weasyprint", _wp)
|
||||||
|
|
||||||
|
from fastapi import FastAPI
|
||||||
|
from fastapi.testclient import TestClient
|
||||||
|
|
||||||
|
from app.api.v1 import admin as admin_module
|
||||||
|
from app.core.db import get_db
|
||||||
|
|
||||||
|
app = FastAPI()
|
||||||
|
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||||||
|
|
||||||
|
fake_rows = [
|
||||||
|
{"source": "avito", "request_delay_sec": 7.0, "description": "Avito", "updated_at": None},
|
||||||
|
{"source": "global", "request_delay_sec": 0.0, "description": "Global", "updated_at": None},
|
||||||
|
]
|
||||||
|
|
||||||
|
mock_db = MagicMock()
|
||||||
|
mock_db.execute.return_value.mappings.return_value.all.return_value = fake_rows
|
||||||
|
app.dependency_overrides[get_db] = lambda: mock_db
|
||||||
|
|
||||||
|
client = TestClient(app)
|
||||||
|
r = client.get("/api/v1/admin/scraper-settings")
|
||||||
|
assert r.status_code == 200
|
||||||
|
data = r.json()
|
||||||
|
assert "settings" in data
|
||||||
|
sources = [s["source"] for s in data["settings"]]
|
||||||
|
assert "global" in sources
|
||||||
|
assert "avito" in sources
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue