feat(tradein): global scraper delay setting (applies across all scrapers) (#485)
Add global request delay floor + per-source delay management. - 054: seed 'global'=0 + per-source rows (avito=7, cian=5, n1=5, domrf=5, rosreestr=5) using ON CONFLICT DO NOTHING; reuses existing 053 table from #484 - scraper_settings.py: get_scraper_delay() = max(per_source, global); _GLOBAL_KEY, refactored _get_setting_cached(); preserved yandex umbrella alias logic - admin.py: GET /scraper-settings + PUT /scraper-settings/{source} (ge=0.0, le=60.0); cache invalidation on update; CAST(:d AS numeric) per psycopg v3 - avito/cian/n1 scrapers wired (yandex was wired by #484) - 18 unit tests pass (10 original updated for max() + 8 new global delay tests) Closes follow-up gap from #486 (Cian admin slider min=0 now accepted by API).
This commit is contained in:
parent
ebfd29a0a6
commit
e79c9cec88
7 changed files with 399 additions and 144 deletions
|
|
@ -79,17 +79,6 @@ class ScrapeResponse(BaseModel):
|
|||
by_source: list[ScrapeResult]
|
||||
|
||||
|
||||
class ScraperSetting(BaseModel):
|
||||
source: str
|
||||
request_delay_sec: float
|
||||
description: str | None = None
|
||||
updated_at: str | None = None # ISO
|
||||
|
||||
|
||||
class ScraperSettingUpdate(BaseModel):
|
||||
request_delay_sec: float = Field(ge=1.0, le=60.0)
|
||||
|
||||
|
||||
@router.post("/scrape", response_model=ScrapeResponse)
|
||||
async def scrape_around(
|
||||
payload: ScrapeRequest,
|
||||
|
|
@ -560,6 +549,74 @@ def cancel_avito_city_sweep(
|
|||
return {"ok": True, "run_id": run_id, "cancelled": cancelled}
|
||||
|
||||
|
||||
# ── Scraper settings: global + per-source delay management ───────────────────
|
||||
|
||||
|
||||
class ScraperSettingPayload(BaseModel):
|
||||
source: str = Field(..., min_length=1, max_length=64)
|
||||
request_delay_sec: float = Field(..., ge=0.0, le=60.0)
|
||||
description: str | None = None
|
||||
|
||||
|
||||
@router.get("/scraper-settings", status_code=200)
|
||||
def list_scraper_settings(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict:
|
||||
"""Список всех настроек задержки (per-source + global)."""
|
||||
rows = db.execute(
|
||||
text("""
|
||||
SELECT source, request_delay_sec, description, updated_at
|
||||
FROM scraper_settings
|
||||
ORDER BY source ASC
|
||||
""")
|
||||
).mappings().all()
|
||||
return {"settings": [dict(r) for r in rows]}
|
||||
|
||||
|
||||
@router.put("/scraper-settings/{source}", status_code=200)
|
||||
def update_scraper_setting(
|
||||
source: str,
|
||||
payload: ScraperSettingPayload,
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> dict:
|
||||
"""Обновить задержку для source (или 'global'). Path source используется как ключ.
|
||||
|
||||
Эффект применяется немедленно — invalidate_cache() сбрасывает кеш для source,
|
||||
следующий get_scraper_delay() перечитает из БД.
|
||||
|
||||
source='global' — нижняя планка для всех парсеров (0 = отключено).
|
||||
"""
|
||||
if payload.source != source:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"Path source={source!r} does not match body source={payload.source!r}",
|
||||
)
|
||||
|
||||
row = db.execute(
|
||||
text("""
|
||||
INSERT INTO scraper_settings (source, request_delay_sec, description, updated_at)
|
||||
VALUES (:s, CAST(:d AS numeric), :desc, NOW())
|
||||
ON CONFLICT (source) DO UPDATE SET
|
||||
request_delay_sec = EXCLUDED.request_delay_sec,
|
||||
description = COALESCE(EXCLUDED.description, scraper_settings.description),
|
||||
updated_at = NOW()
|
||||
RETURNING source, request_delay_sec, description, updated_at
|
||||
"""),
|
||||
{
|
||||
"s": source,
|
||||
"d": payload.request_delay_sec,
|
||||
"desc": payload.description,
|
||||
},
|
||||
).mappings().one()
|
||||
db.commit()
|
||||
|
||||
invalidate_cache(source)
|
||||
logger.info(
|
||||
"scraper-settings: updated source=%s delay=%.1f", source, payload.request_delay_sec
|
||||
)
|
||||
return dict(row)
|
||||
|
||||
|
||||
# ── In-app scheduler endpoints (Stage 4e) ────────────────────────────────────
|
||||
|
||||
|
||||
|
|
@ -649,78 +706,6 @@ def update_schedule(
|
|||
)
|
||||
|
||||
|
||||
# -- Scraper settings (per-source live config) --------------------------------
|
||||
|
||||
|
||||
@router.get("/scraper-settings", response_model=list[ScraperSetting])
|
||||
async def list_scraper_settings(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> list[ScraperSetting]:
|
||||
"""Return all scraper_settings rows (live config for delays)."""
|
||||
rows = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT source, request_delay_sec, description, updated_at
|
||||
FROM scraper_settings
|
||||
ORDER BY source
|
||||
"""
|
||||
)
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
ScraperSetting(
|
||||
source=r["source"],
|
||||
request_delay_sec=float(r["request_delay_sec"]),
|
||||
description=r["description"],
|
||||
updated_at=r["updated_at"].isoformat() if r["updated_at"] else None,
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
@router.put("/scraper-settings/{source}", response_model=ScraperSetting)
|
||||
async def update_scraper_setting(
|
||||
source: str,
|
||||
payload: ScraperSettingUpdate,
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
) -> ScraperSetting:
|
||||
"""Update request_delay_sec for one source. UPSERT - creates row if missing.
|
||||
|
||||
Invalidates the in-process cache so the change takes effect on next
|
||||
scraper instantiation (usually within seconds).
|
||||
"""
|
||||
row = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO scraper_settings (source, request_delay_sec, updated_at)
|
||||
VALUES (:source, :delay, NOW())
|
||||
ON CONFLICT (source) DO UPDATE
|
||||
SET request_delay_sec = EXCLUDED.request_delay_sec,
|
||||
updated_at = NOW()
|
||||
RETURNING source, request_delay_sec, description, updated_at
|
||||
"""
|
||||
),
|
||||
{"source": source, "delay": payload.request_delay_sec},
|
||||
)
|
||||
.mappings()
|
||||
.first()
|
||||
)
|
||||
db.commit()
|
||||
invalidate_cache(source)
|
||||
if row is None:
|
||||
raise HTTPException(500, "Failed to upsert scraper_settings row")
|
||||
return ScraperSetting(
|
||||
source=row["source"],
|
||||
request_delay_sec=float(row["request_delay_sec"]),
|
||||
description=row["description"],
|
||||
updated_at=row["updated_at"].isoformat() if row["updated_at"] else None,
|
||||
)
|
||||
|
||||
|
||||
# -- Yandex ad-hoc scrape triggers --------------------------------------------
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,12 @@
|
|||
"""Runtime config loader for scraper delays.
|
||||
"""scraper_settings.py — загрузка задержек парсеров из таблицы scraper_settings.
|
||||
|
||||
Backed by scraper_settings table with 60s in-process cache.
|
||||
Кеш в памяти с TTL 60 секунд — не нагружает БД при каждом запросе.
|
||||
invalidate_cache() вызывается из admin API PUT для немедленного применения.
|
||||
|
||||
Ключевая логика:
|
||||
get_scraper_delay(source) = max(per_source_delay, global_delay)
|
||||
Строка source='global' задаёт нижнюю планку для ВСЕХ парсеров.
|
||||
Если global=0 — используется только per-source значение.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -13,6 +19,9 @@ from sqlalchemy import text
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Специальный ключ глобальной задержки (строка в scraper_settings с этим source).
|
||||
_GLOBAL_KEY = "global"
|
||||
|
||||
# In-process cache: {source: (delay_sec, fetched_at_epoch)}
|
||||
_CACHE: dict[str, tuple[float, float]] = {}
|
||||
_CACHE_LOCK = Lock()
|
||||
|
|
@ -20,10 +29,15 @@ _CACHE_TTL_SEC = 60.0
|
|||
|
||||
# Default fallback if DB row missing or DB unreachable
|
||||
_DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
|
||||
"avito": 7.0,
|
||||
"cian": 5.0,
|
||||
"n1": 5.0,
|
||||
"yandex": 5.0,
|
||||
"yandex_detail": 5.0,
|
||||
"yandex_newbuilding": 5.0,
|
||||
"yandex_valuation": 5.0,
|
||||
"domrf": 5.0,
|
||||
"rosreestr": 5.0,
|
||||
}
|
||||
_GLOBAL_DEFAULT_DELAY = 5.0
|
||||
|
||||
|
|
@ -52,15 +66,13 @@ def _open_session() -> object:
|
|||
return SessionLocal()
|
||||
|
||||
|
||||
def get_scraper_delay(source: str) -> float:
|
||||
"""Return the configured request_delay_sec for `source`.
|
||||
def _get_setting_cached(key: str) -> float:
|
||||
"""Cache-aware read of one source key from scraper_settings.
|
||||
|
||||
- Maps Yandex sub-scrapers to the umbrella 'yandex' key.
|
||||
- Cached 60s in-process to avoid per-instance DB query.
|
||||
- Any DB error -> returns the hardcoded default for that source.
|
||||
Returns request_delay_sec from DB (with TTL=60s cache).
|
||||
Missing row: falls back to _DEFAULT_DELAY_BY_SOURCE, or 0.0 for _GLOBAL_KEY
|
||||
(meaning no global floor is applied when the row hasn't been seeded yet).
|
||||
"""
|
||||
key = _KEY_ALIASES.get(source, source)
|
||||
|
||||
now = time.time()
|
||||
with _CACHE_LOCK:
|
||||
cached = _CACHE.get(key)
|
||||
|
|
@ -77,17 +89,37 @@ def get_scraper_delay(source: str) -> float:
|
|||
).first()
|
||||
if row is not None:
|
||||
value = float(row[0])
|
||||
elif key == _GLOBAL_KEY:
|
||||
# Global row not yet seeded — don't apply floor.
|
||||
value = 0.0
|
||||
else:
|
||||
value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
||||
except Exception as e:
|
||||
logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
|
||||
value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
||||
value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(
|
||||
key, _GLOBAL_DEFAULT_DELAY
|
||||
)
|
||||
|
||||
with _CACHE_LOCK:
|
||||
_CACHE[key] = (value, now)
|
||||
return value
|
||||
|
||||
|
||||
def get_scraper_delay(source: str) -> float:
|
||||
"""Return the configured request_delay_sec for `source`.
|
||||
|
||||
Effective delay = max(per_source, global).
|
||||
- Maps Yandex sub-scrapers to the umbrella 'yandex' key.
|
||||
- Cached 60s in-process to avoid per-instance DB query.
|
||||
- source='global' sets a floor across all scrapers (0 = disabled).
|
||||
- Any DB error -> returns the hardcoded default for that source.
|
||||
"""
|
||||
key = _KEY_ALIASES.get(source, source)
|
||||
per_source = _get_setting_cached(key)
|
||||
global_value = _get_setting_cached(_GLOBAL_KEY)
|
||||
return max(per_source, global_value)
|
||||
|
||||
|
||||
def invalidate_cache(source: str | None = None) -> None:
|
||||
"""Clear the cache entry for `source`, or the whole cache if None.
|
||||
|
||||
|
|
@ -96,6 +128,11 @@ def invalidate_cache(source: str | None = None) -> None:
|
|||
with _CACHE_LOCK:
|
||||
if source is None:
|
||||
_CACHE.clear()
|
||||
logger.info("scraper_settings: cache cleared (all sources)")
|
||||
else:
|
||||
key = _KEY_ALIASES.get(source, source)
|
||||
_CACHE.pop(key, None)
|
||||
# Also clear the source itself if it differed from key (alias case).
|
||||
if key != source:
|
||||
_CACHE.pop(source, None)
|
||||
logger.info("scraper_settings: cache cleared for source=%s", source)
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from urllib.parse import urlencode, urljoin
|
|||
from curl_cffi.requests import AsyncSession
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -36,11 +37,13 @@ class AvitoScraper(BaseScraper):
|
|||
|
||||
name = "avito"
|
||||
base_url = "https://www.avito.ru"
|
||||
# Avito жёстко мониторит — спим долго между запросами
|
||||
# Avito жёстко мониторит — спим долго между запросами.
|
||||
# Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра.
|
||||
request_delay_sec = 7.0
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
self._cffi: AsyncSession | None = None
|
||||
|
||||
async def __aenter__(self) -> AvitoScraper:
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from urllib.parse import urlencode
|
|||
|
||||
from curl_cffi.requests import AsyncSession
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
from app.services.scrapers.cian_state_parser import extract_state
|
||||
|
||||
|
|
@ -46,10 +47,12 @@ class CianScraper(BaseScraper):
|
|||
name = "cian"
|
||||
# ekb.cian.ru — city-specific subdomain для ЕКБ (per Schema sec 13, closed Q4)
|
||||
base_url = "https://ekb.cian.ru"
|
||||
# Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра.
|
||||
request_delay_sec = 5.0 # консервативно: Cian менее агрессивен чем Avito, но 5s безопасно
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
self._cffi: AsyncSession | None = None
|
||||
|
||||
async def __aenter__(self) -> CianScraper:
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from typing import Any
|
|||
from curl_cffi.requests import AsyncSession
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
|
@ -35,13 +36,15 @@ class N1Scraper(BaseScraper):
|
|||
|
||||
name = "n1"
|
||||
base_url = "https://ekaterinburg.n1.ru"
|
||||
# Класс-дефолт; реальное значение загружается из scraper_settings при создании экземпляра.
|
||||
request_delay_sec = 5.0
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
self._cffi: AsyncSession | None = None
|
||||
|
||||
async def __aenter__(self) -> "N1Scraper":
|
||||
async def __aenter__(self) -> N1Scraper:
|
||||
await super().__aenter__()
|
||||
self._cffi = AsyncSession(
|
||||
impersonate="chrome120",
|
||||
|
|
@ -74,7 +77,7 @@ class N1Scraper(BaseScraper):
|
|||
try:
|
||||
assert self._cffi is not None
|
||||
response = await self._cffi.get(url)
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
logger.exception("n1 fetch failed for %s", url)
|
||||
return []
|
||||
if response.status_code != 200:
|
||||
|
|
@ -153,7 +156,9 @@ class N1Scraper(BaseScraper):
|
|||
title = link.text(strip=True) or ""
|
||||
rooms = _extract_rooms(title)
|
||||
# Адрес = title без префикса комнат
|
||||
address = re.sub(r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE).strip()
|
||||
address = re.sub(
|
||||
r"^\s*(\d+-к|студи[яи])\s*,?\s*", "", title, flags=re.IGNORECASE
|
||||
).strip()
|
||||
if not address:
|
||||
address = "Екатеринбург (N1)"
|
||||
|
||||
|
|
@ -192,7 +197,7 @@ class N1Scraper(BaseScraper):
|
|||
photo_urls=photos,
|
||||
raw_payload={"title": title},
|
||||
)
|
||||
except Exception: # noqa: BLE001
|
||||
except Exception:
|
||||
logger.exception("n1 card parse failed")
|
||||
return None
|
||||
|
||||
|
|
|
|||
25
tradein-mvp/backend/data/sql/054_scraper_settings_global.sql
Normal file
25
tradein-mvp/backend/data/sql/054_scraper_settings_global.sql
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
-- 054_scraper_settings_global.sql
|
||||
-- Add global delay row + per-source aliases for all scrapers.
|
||||
-- Idempotent — ON CONFLICT DO NOTHING preserves user-edited values.
|
||||
BEGIN;
|
||||
|
||||
-- 'global' row: when > 0, acts as floor (max with per-source delay).
|
||||
-- Default 0 = use per-source defaults.
|
||||
INSERT INTO scraper_settings (source, request_delay_sec, description)
|
||||
VALUES (
|
||||
'global',
|
||||
0.0,
|
||||
'Global delay applied across ALL scrapers (max with per-source). 0 = use per-source defaults.'
|
||||
)
|
||||
ON CONFLICT (source) DO NOTHING;
|
||||
|
||||
-- Per-source rows for all scrapers (individual control).
|
||||
INSERT INTO scraper_settings (source, request_delay_sec, description) VALUES
|
||||
('avito', 7.0, 'Avito SERP delay (anti-ban)'),
|
||||
('cian', 5.0, 'Cian SERP delay'),
|
||||
('n1', 5.0, 'N1 SERP delay'),
|
||||
('domrf', 5.0, 'DOM.RF SERP delay'),
|
||||
('rosreestr', 5.0, 'Rosreestr API delay')
|
||||
ON CONFLICT (source) DO NOTHING;
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -1,14 +1,30 @@
|
|||
"""Tests for scraper_settings loader (cache + DB fallback + alias mapping)."""
|
||||
"""Tests for scraper_settings loader (cache + DB fallback + alias mapping + global delay).
|
||||
|
||||
Covers:
|
||||
- Per-source value lookup + caching (main's tests)
|
||||
- Global delay max() logic (our additions)
|
||||
- Admin endpoint smoke
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
|
||||
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
|
||||
import app.services.scraper_settings as ss_mod
|
||||
from app.services import scraper_settings as ss
|
||||
from app.services.scraper_settings import get_scraper_delay, invalidate_cache
|
||||
|
||||
# Patch target: _open_session is a module-level wrapper so tests can mock DB
|
||||
_PATCH_TARGET = "app.services.scraper_settings._open_session"
|
||||
|
||||
|
||||
# ── Fixtures ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_cache():
|
||||
ss._CACHE.clear()
|
||||
|
|
@ -16,8 +32,15 @@ def _reset_cache():
|
|||
ss._CACHE.clear()
|
||||
|
||||
|
||||
def _mock_session(row_value: float | None) -> MagicMock:
|
||||
"""Return a mock that behaves like SessionLocal() context-manager."""
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _mock_session_uniform(row_value: float | None) -> tuple[MagicMock, MagicMock]:
|
||||
"""Mock that returns the same value for any key (context-manager style).
|
||||
|
||||
Used by tests that don't care about multi-key distinction.
|
||||
Global key will also return the same value, so max() returns row_value.
|
||||
"""
|
||||
session = MagicMock()
|
||||
session.__enter__ = MagicMock(return_value=session)
|
||||
session.__exit__ = MagicMock(return_value=False)
|
||||
|
|
@ -25,56 +48,86 @@ def _mock_session(row_value: float | None) -> MagicMock:
|
|||
session.execute.return_value.first.return_value = None
|
||||
else:
|
||||
session.execute.return_value.first.return_value = (row_value,)
|
||||
# _open_session() returns the session directly (no extra call)
|
||||
mock_open = MagicMock(return_value=session)
|
||||
return mock_open, session
|
||||
|
||||
|
||||
def _make_mock_session(values: dict[str, float | None]) -> MagicMock:
|
||||
"""Mock сессии БД с фиксированными значениями per source key.
|
||||
|
||||
Поддерживает разные значения для разных ключей — используется в тестах
|
||||
global delay логики.
|
||||
"""
|
||||
mock_db = MagicMock()
|
||||
|
||||
def fake_execute(sql, params):
|
||||
key = params["s"]
|
||||
result = MagicMock()
|
||||
val = values.get(key)
|
||||
result.first.return_value = (val,) if val is not None else None
|
||||
return result
|
||||
|
||||
mock_db.execute = fake_execute
|
||||
mock_db.__enter__ = MagicMock(return_value=mock_db)
|
||||
mock_db.__exit__ = MagicMock(return_value=False)
|
||||
return mock_db
|
||||
|
||||
|
||||
# ── Main's tests: core cache + alias + fallback behaviour ─────────────────────
|
||||
|
||||
|
||||
def test_returns_db_value_for_yandex():
|
||||
mock_open, _ = _mock_session(7.5)
|
||||
"""DB value 7.5, global also 7.5 → max(7.5, 7.5) = 7.5."""
|
||||
mock_open, _ = _mock_session_uniform(7.5)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
assert ss.get_scraper_delay("yandex") == 7.5
|
||||
|
||||
|
||||
def test_yandex_subkeys_map_to_yandex_umbrella():
|
||||
"""yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row."""
|
||||
mock_open, _session = _mock_session(8.0)
|
||||
mock_open, _session = _mock_session_uniform(8.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
assert ss.get_scraper_delay("yandex_detail") == 8.0
|
||||
assert ss.get_scraper_delay("yandex_newbuilding") == 8.0
|
||||
assert ss.get_scraper_delay("yandex_valuation") == 8.0
|
||||
# All 3 lookups hit the cache after the first; _open_session called once
|
||||
assert mock_open.call_count == 1
|
||||
|
||||
|
||||
def test_cache_hit_within_ttl():
|
||||
mock_open, _session = _mock_session(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex")
|
||||
ss.get_scraper_delay("yandex")
|
||||
ss.get_scraper_delay("yandex")
|
||||
assert mock_open.call_count == 1 # cached after first
|
||||
|
||||
|
||||
def test_cache_expires_after_ttl():
|
||||
mock_open, _session = _mock_session(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex")
|
||||
# Force-age the cache
|
||||
with ss._CACHE_LOCK:
|
||||
v, t = ss._CACHE["yandex"]
|
||||
ss._CACHE["yandex"] = (v, t - 999)
|
||||
ss.get_scraper_delay("yandex")
|
||||
# First call fetches yandex + global (2 DB calls). Remaining 2 subkey calls
|
||||
# map to "yandex" (already cached) and "global" (already cached) → 0 more.
|
||||
assert mock_open.call_count == 2
|
||||
|
||||
|
||||
def test_cache_hit_within_ttl():
|
||||
"""After first call both yandex+global are cached; repeats hit cache only."""
|
||||
mock_open, _session = _mock_session_uniform(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex")
|
||||
ss.get_scraper_delay("yandex")
|
||||
ss.get_scraper_delay("yandex")
|
||||
# First call: 2 DB hits (yandex + global). Subsequent: 0.
|
||||
assert mock_open.call_count == 2
|
||||
|
||||
|
||||
def test_cache_expires_after_ttl():
|
||||
"""Force-age yandex key — next call re-fetches yandex only (global still fresh)."""
|
||||
mock_open, _session = _mock_session_uniform(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex") # +2 calls (yandex + global)
|
||||
# Force-age the yandex entry
|
||||
with ss._CACHE_LOCK:
|
||||
v, t = ss._CACHE["yandex"]
|
||||
ss._CACHE["yandex"] = (v, t - 999)
|
||||
ss.get_scraper_delay("yandex") # +1 call (yandex stale; global still fresh)
|
||||
assert mock_open.call_count == 3
|
||||
|
||||
|
||||
def test_db_missing_row_returns_default():
|
||||
mock_open, _ = _mock_session(None)
|
||||
"""Missing yandex row → _DEFAULT_DELAY_BY_SOURCE. Global None → 0.0. max = default."""
|
||||
mock_open, _ = _mock_session_uniform(None)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]
|
||||
|
||||
|
||||
def test_db_error_returns_default():
|
||||
"""DB error → hardcoded default. Global error → 0.0. max = default."""
|
||||
bad_session = MagicMock()
|
||||
bad_session.__enter__ = MagicMock(return_value=bad_session)
|
||||
bad_session.__exit__ = MagicMock(return_value=False)
|
||||
|
|
@ -85,34 +138,178 @@ def test_db_error_returns_default():
|
|||
|
||||
|
||||
def test_unknown_source_uses_global_default():
|
||||
mock_open, _ = _mock_session(None)
|
||||
"""Unknown source, DB row None → _GLOBAL_DEFAULT_DELAY. Global None → 0.0. max = 5.0."""
|
||||
mock_open, _ = _mock_session_uniform(None)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY
|
||||
|
||||
|
||||
def test_invalidate_cache_specific_source():
|
||||
mock_open, _ = _mock_session(6.0)
|
||||
"""invalidate_cache(yandex) drops yandex entry; next call re-fetches yandex only."""
|
||||
mock_open, _ = _mock_session_uniform(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex")
|
||||
ss.invalidate_cache("yandex")
|
||||
ss.get_scraper_delay("yandex")
|
||||
assert mock_open.call_count == 2
|
||||
ss.get_scraper_delay("yandex") # +2 (yandex + global)
|
||||
ss.invalidate_cache("yandex") # clears yandex key
|
||||
ss.get_scraper_delay("yandex") # +1 (yandex stale; global still cached)
|
||||
assert mock_open.call_count == 3
|
||||
|
||||
|
||||
def test_invalidate_cache_all():
|
||||
mock_open, _ = _mock_session(6.0)
|
||||
"""invalidate_cache() clears all entries; next call re-fetches both."""
|
||||
mock_open, _ = _mock_session_uniform(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex")
|
||||
ss.invalidate_cache()
|
||||
ss.get_scraper_delay("yandex")
|
||||
assert mock_open.call_count == 2
|
||||
ss.get_scraper_delay("yandex") # +2 (yandex + global)
|
||||
ss.invalidate_cache() # clears everything
|
||||
ss.get_scraper_delay("yandex") # +2 (both stale)
|
||||
assert mock_open.call_count == 4
|
||||
|
||||
|
||||
def test_invalidate_alias_invalidates_umbrella():
|
||||
"""invalidate_cache('yandex_detail') should clear the 'yandex' umbrella key."""
|
||||
mock_open, _ = _mock_session(6.0)
|
||||
"""invalidate_cache('yandex_detail') clears the 'yandex' umbrella key."""
|
||||
mock_open, _ = _mock_session_uniform(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex_detail")
|
||||
ss.invalidate_cache("yandex_detail")
|
||||
ss.get_scraper_delay("yandex_detail")
|
||||
assert mock_open.call_count == 2
|
||||
ss.get_scraper_delay("yandex_detail") # +2 (yandex + global)
|
||||
ss.invalidate_cache("yandex_detail") # clears yandex (via alias)
|
||||
ss.get_scraper_delay("yandex_detail") # +1 (yandex stale; global fresh)
|
||||
assert mock_open.call_count == 3
|
||||
|
||||
|
||||
# ── Our additions: global delay max() logic ───────────────────────────────────
|
||||
|
||||
|
||||
def test_global_delay_takes_precedence_when_higher(monkeypatch):
|
||||
"""global=10 > avito=7 → returns 10."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"avito": 7.0, "global": 10.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
delay = get_scraper_delay("avito")
|
||||
assert delay == 10.0
|
||||
|
||||
|
||||
def test_per_source_takes_precedence_when_higher(monkeypatch):
|
||||
"""avito=7 > global=2 → returns 7."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"avito": 7.0, "global": 2.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
delay = get_scraper_delay("avito")
|
||||
assert delay == 7.0
|
||||
|
||||
|
||||
def test_global_zero_means_no_floor(monkeypatch):
|
||||
"""global=0 → only per-source value applies."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"cian": 5.0, "global": 0.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
delay = get_scraper_delay("cian")
|
||||
assert delay == 5.0
|
||||
|
||||
|
||||
def test_missing_per_source_row_uses_class_default(monkeypatch):
|
||||
"""Missing source row → _DEFAULT_DELAY_BY_SOURCE. global=0 → no floor."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"n1": None, "global": 0.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
delay = get_scraper_delay("n1")
|
||||
assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["n1"]
|
||||
|
||||
|
||||
def test_global_missing_row_treated_as_zero(monkeypatch):
|
||||
"""Missing 'global' row → fallback 0.0, avito=7.0 → max(7.0, 0.0) = 7.0."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"avito": 7.0, "global": None})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
delay = get_scraper_delay("avito")
|
||||
assert delay == 7.0
|
||||
|
||||
|
||||
def test_db_error_falls_back_to_default_global_logic(monkeypatch):
|
||||
"""DB error → graceful fallback, no crash. avito default=7.0, global error→0.0."""
|
||||
invalidate_cache()
|
||||
|
||||
def broken_open():
|
||||
raise RuntimeError("DB is down")
|
||||
|
||||
monkeypatch.setattr(ss_mod, "_open_session", broken_open)
|
||||
|
||||
delay = get_scraper_delay("avito")
|
||||
assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["avito"]
|
||||
|
||||
|
||||
def test_invalidate_cache_single_source_global_logic(monkeypatch):
|
||||
"""invalidate_cache(source) removes only that source from cache."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
get_scraper_delay("cian")
|
||||
get_scraper_delay("avito")
|
||||
|
||||
invalidate_cache("cian")
|
||||
|
||||
with ss_mod._CACHE_LOCK:
|
||||
assert "cian" not in ss_mod._CACHE
|
||||
assert "avito" in ss_mod._CACHE
|
||||
|
||||
|
||||
def test_invalidate_cache_all_global_logic(monkeypatch):
|
||||
"""invalidate_cache() without args clears the entire cache."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"avito": 7.0, "global": 0.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
get_scraper_delay("avito")
|
||||
invalidate_cache()
|
||||
|
||||
with ss_mod._CACHE_LOCK:
|
||||
assert len(ss_mod._CACHE) == 0
|
||||
|
||||
|
||||
# ── Admin API endpoint smoke ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_list_scraper_settings_endpoint(monkeypatch):
|
||||
"""GET /scraper-settings mounts and returns a list including 'global'."""
|
||||
import sys
|
||||
|
||||
# Stub weasyprint (not on CI)
|
||||
_wp = MagicMock()
|
||||
sys.modules.setdefault("weasyprint", _wp)
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.v1 import admin as admin_module
|
||||
from app.core.db import get_db
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||||
|
||||
fake_rows = [
|
||||
{"source": "avito", "request_delay_sec": 7.0, "description": "Avito", "updated_at": None},
|
||||
{"source": "global", "request_delay_sec": 0.0, "description": "Global", "updated_at": None},
|
||||
]
|
||||
|
||||
mock_db = MagicMock()
|
||||
mock_db.execute.return_value.mappings.return_value.all.return_value = fake_rows
|
||||
app.dependency_overrides[get_db] = lambda: mock_db
|
||||
|
||||
client = TestClient(app)
|
||||
r = client.get("/api/v1/admin/scraper-settings")
|
||||
assert r.status_code == 200
|
||||
data = r.json()
|
||||
assert "settings" in data
|
||||
sources = [s["source"] for s in data["settings"]]
|
||||
assert "global" in sources
|
||||
assert "avito" in sources
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue