Merge remote-tracking branch 'forgejo/main' into feat/tradein-global-scraper-delay
# Conflicts: # tradein-mvp/backend/app/services/scraper_settings.py # tradein-mvp/backend/data/sql/053_scraper_settings.sql # tradein-mvp/backend/tests/test_scraper_settings.py
This commit is contained in:
lekss361
commit
814530d187
9 changed files with 447 additions and 136 deletions
|
|
@ -35,7 +35,10 @@ from app.services.scrapers.avito_imv import (
|
|||
from app.services.scrapers.base import save_listings
|
||||
from app.services.scrapers.cian import CianScraper
|
||||
from app.services.scrapers.n1 import N1Scraper
|
||||
from app.services.scrapers.yandex_detail import YandexDetailScraper
|
||||
from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
|
||||
from app.services.scrapers.yandex_realty import YandexRealtyScraper
|
||||
from app.services.scrapers.yandex_valuation import YandexValuationScraper
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
|
@ -701,3 +704,122 @@ def update_schedule(
|
|||
next_run_at=row["next_run_at"].isoformat() if row["next_run_at"] else None,
|
||||
updated_at=row["updated_at"].isoformat() if row["updated_at"] else None,
|
||||
)
|
||||
|
||||
|
||||
# -- Yandex ad-hoc scrape triggers --------------------------------------------
|
||||
|
||||
|
||||
class YandexDetailTriggerResp(BaseModel):
|
||||
ok: bool
|
||||
offer_url: str
|
||||
offer_id: str | None
|
||||
price_rub: int | None
|
||||
title: str | None
|
||||
photo_count: int
|
||||
|
||||
|
||||
@router.post("/scrape/yandex-detail", response_model=YandexDetailTriggerResp)
|
||||
async def scrape_yandex_detail(
|
||||
offer_url: str,
|
||||
) -> YandexDetailTriggerResp:
|
||||
"""Ad-hoc parse one Yandex offer detail page.
|
||||
|
||||
Returns a snapshot of the extracted DetailEnrichment (no DB write - read-only
|
||||
debug; main pipeline writes via estimator on /estimate flow).
|
||||
"""
|
||||
async with YandexDetailScraper() as scraper:
|
||||
result = await scraper.fetch_detail(offer_url)
|
||||
if result is None:
|
||||
raise HTTPException(404, f"Could not parse Yandex detail page: {offer_url}")
|
||||
return YandexDetailTriggerResp(
|
||||
ok=True,
|
||||
offer_url=offer_url,
|
||||
offer_id=result.offer_id,
|
||||
price_rub=result.price_rub,
|
||||
title=result.title,
|
||||
photo_count=len(result.photo_urls),
|
||||
)
|
||||
|
||||
|
||||
class YandexNewbuildingTriggerResp(BaseModel):
|
||||
ok: bool
|
||||
ext_id: str
|
||||
ext_slug: str
|
||||
name: str | None
|
||||
lat: float | None
|
||||
lon: float | None
|
||||
rating: float | None
|
||||
ratings_count: int | None
|
||||
text_reviews_count: int | None
|
||||
developer_name: str | None
|
||||
|
||||
|
||||
@router.post("/scrape/yandex-newbuilding", response_model=YandexNewbuildingTriggerResp)
|
||||
async def scrape_yandex_newbuilding(
|
||||
slug: str,
|
||||
id: str,
|
||||
city: str = "ekaterinburg",
|
||||
) -> YandexNewbuildingTriggerResp:
|
||||
"""Ad-hoc parse a Yandex JK landing page.
|
||||
|
||||
URL: /{city}/kupit/novostrojka/<slug>-<id>/
|
||||
"""
|
||||
async with YandexNewbuildingScraper() as scraper:
|
||||
result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city)
|
||||
if result is None:
|
||||
raise HTTPException(404, f"Could not parse Yandex JK: {slug}-{id} in {city}")
|
||||
return YandexNewbuildingTriggerResp(
|
||||
ok=True,
|
||||
ext_id=result.ext_id,
|
||||
ext_slug=result.ext_slug,
|
||||
name=result.name,
|
||||
lat=result.lat,
|
||||
lon=result.lon,
|
||||
rating=result.rating,
|
||||
ratings_count=result.ratings_count,
|
||||
text_reviews_count=result.text_reviews_count,
|
||||
developer_name=result.developer_name,
|
||||
)
|
||||
|
||||
|
||||
class YandexValuationTriggerResp(BaseModel):
|
||||
ok: bool
|
||||
address: str
|
||||
year_built: int | None
|
||||
total_floors: int | None
|
||||
house_type: str | None
|
||||
has_lift: bool | None
|
||||
total_objects: int | None
|
||||
history_items_count: int
|
||||
|
||||
|
||||
@router.post("/scrape/yandex-valuation", response_model=YandexValuationTriggerResp)
|
||||
async def scrape_yandex_valuation(
|
||||
address: str,
|
||||
offer_category: str = "APARTMENT",
|
||||
offer_type: str = "SELL",
|
||||
page: int = 1,
|
||||
) -> YandexValuationTriggerResp:
|
||||
"""Ad-hoc fetch Yandex Valuation house-history for an address (debug, no cache write).
|
||||
|
||||
Anonymous GET - works without auth.
|
||||
"""
|
||||
async with YandexValuationScraper() as scraper:
|
||||
result = await scraper.fetch_house_history(
|
||||
address=address,
|
||||
offer_category=offer_category,
|
||||
offer_type=offer_type,
|
||||
page=page,
|
||||
)
|
||||
if result is None:
|
||||
raise HTTPException(404, f"Could not parse Yandex Valuation: {address}")
|
||||
return YandexValuationTriggerResp(
|
||||
ok=True,
|
||||
address=address,
|
||||
year_built=result.house.year_built,
|
||||
total_floors=result.house.total_floors,
|
||||
house_type=result.house.house_type,
|
||||
has_lift=result.house.has_lift,
|
||||
total_objects=result.house.total_objects,
|
||||
history_items_count=len(result.history_items),
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,60 +12,66 @@ invalidate_cache() вызывается из admin API PUT для немедле
|
|||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Generator
|
||||
from contextlib import contextmanager
|
||||
from threading import Lock
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Специальный ключ глобальной задержки (строка в scraper_settings с этим source).
|
||||
_GLOBAL_KEY = "global"
|
||||
|
||||
# TTL кеша в секундах — после истечения перечитывается из БД.
|
||||
# In-process cache: {source: (delay_sec, fetched_at_epoch)}
|
||||
_CACHE: dict[str, tuple[float, float]] = {}
|
||||
_CACHE_LOCK = Lock()
|
||||
_CACHE_TTL_SEC = 60.0
|
||||
|
||||
# Дефолтные задержки если строка в БД отсутствует.
|
||||
# Default fallback if DB row missing or DB unreachable
|
||||
_DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
|
||||
"avito": 7.0,
|
||||
"cian": 5.0,
|
||||
"n1": 5.0,
|
||||
"yandex": 5.0,
|
||||
"yandex_detail": 5.0,
|
||||
"yandex_newbuilding": 5.0,
|
||||
"yandex_valuation": 5.0,
|
||||
"domrf": 5.0,
|
||||
"rosreestr": 5.0,
|
||||
}
|
||||
|
||||
# Fallback для неизвестного источника.
|
||||
_GLOBAL_DEFAULT_DELAY = 5.0
|
||||
|
||||
# Алиасы источников: ключ → canonical source (можно расширять без изменения вызывающего кода).
|
||||
_KEY_ALIASES: dict[str, str] = {}
|
||||
|
||||
# Кеш: source → (delay_value, timestamp).
|
||||
_CACHE: dict[str, tuple[float, float]] = {}
|
||||
_CACHE_LOCK = threading.Lock()
|
||||
# Yandex sub-scrapers all read the umbrella "yandex" key.
|
||||
# Map: scraper_name -> DB row key.
|
||||
_KEY_ALIASES: dict[str, str] = {
|
||||
"yandex": "yandex",
|
||||
"yandex_detail": "yandex",
|
||||
"yandex_newbuilding": "yandex",
|
||||
"yandex_realty_nb": "yandex", # BaseScraper.name used by YandexNewbuildingScraper
|
||||
"yandex_valuation": "yandex",
|
||||
# Non-Yandex sources resolve to their own keys for forward-compat
|
||||
# (no rows exist yet for them; fall back to default).
|
||||
}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _open_session() -> Generator[Session, None, None]:
|
||||
"""Открыть сессию БД с гарантированным закрытием."""
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
def _open_session() -> object:
|
||||
"""Return a new SessionLocal context-manager instance.
|
||||
|
||||
Deferred import avoids triggering Settings() validation at module import
|
||||
time (unit tests run without DATABASE_URL env var).
|
||||
Extracted as a named function so tests can patch it.
|
||||
"""
|
||||
from app.core.db import SessionLocal
|
||||
|
||||
return SessionLocal()
|
||||
|
||||
|
||||
def _get_setting_cached(key: str) -> float:
|
||||
"""Кеш-aware чтение одного source из scraper_settings.
|
||||
"""Cache-aware read of one source key from scraper_settings.
|
||||
|
||||
Возвращает request_delay_sec из БД (с кешем TTL=60s).
|
||||
При отсутствии строки — фолбек на _DEFAULT_DELAY_BY_SOURCE или 0.0 для global.
|
||||
Returns request_delay_sec from DB (with TTL=60s cache).
|
||||
Missing row: falls back to _DEFAULT_DELAY_BY_SOURCE, or 0.0 for _GLOBAL_KEY
|
||||
(meaning no global floor is applied when the row hasn't been seeded yet).
|
||||
"""
|
||||
now = time.time()
|
||||
with _CACHE_LOCK:
|
||||
|
|
@ -76,20 +82,20 @@ def _get_setting_cached(key: str) -> float:
|
|||
try:
|
||||
with _open_session() as db:
|
||||
row = db.execute(
|
||||
text("SELECT request_delay_sec FROM scraper_settings WHERE source = :s"),
|
||||
text(
|
||||
"SELECT request_delay_sec FROM scraper_settings WHERE source = :s"
|
||||
),
|
||||
{"s": key},
|
||||
).first()
|
||||
if row is not None:
|
||||
value = float(row[0])
|
||||
elif key == _GLOBAL_KEY:
|
||||
# Глобальная строка ещё не создана — не применяем floor.
|
||||
# Global row not yet seeded — don't apply floor.
|
||||
value = 0.0
|
||||
else:
|
||||
value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"scraper_settings: load failed for %s — using default: %s", key, exc
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
|
||||
value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(
|
||||
key, _GLOBAL_DEFAULT_DELAY
|
||||
)
|
||||
|
|
@ -100,10 +106,13 @@ def _get_setting_cached(key: str) -> float:
|
|||
|
||||
|
||||
def get_scraper_delay(source: str) -> float:
|
||||
"""Вернуть эффективную задержку для source = max(per_source, global).
|
||||
"""Return the configured request_delay_sec for `source`.
|
||||
|
||||
Строка source='global' задаёт нижнюю планку для всех парсеров.
|
||||
Если global=0 — используется только per-source значение.
|
||||
Effective delay = max(per_source, global).
|
||||
- Maps Yandex sub-scrapers to the umbrella 'yandex' key.
|
||||
- Cached 60s in-process to avoid per-instance DB query.
|
||||
- source='global' sets a floor across all scrapers (0 = disabled).
|
||||
- Any DB error -> returns the hardcoded default for that source.
|
||||
"""
|
||||
key = _KEY_ALIASES.get(source, source)
|
||||
per_source = _get_setting_cached(key)
|
||||
|
|
@ -112,19 +121,18 @@ def get_scraper_delay(source: str) -> float:
|
|||
|
||||
|
||||
def invalidate_cache(source: str | None = None) -> None:
|
||||
"""Сбросить кеш для source (или весь кеш если source=None).
|
||||
"""Clear the cache entry for `source`, or the whole cache if None.
|
||||
|
||||
Вызывается из admin API PUT /scraper-settings/{source} для немедленного
|
||||
применения нового значения без ожидания TTL.
|
||||
Called by admin endpoint after PUT to force a re-read on next get_scraper_delay().
|
||||
"""
|
||||
with _CACHE_LOCK:
|
||||
if source is None:
|
||||
_CACHE.clear()
|
||||
logger.info("scraper_settings: cache cleared (all sources)")
|
||||
else:
|
||||
_CACHE.pop(source, None)
|
||||
# Если source — алиас, сбрасываем canonical key тоже.
|
||||
canonical = _KEY_ALIASES.get(source)
|
||||
if canonical:
|
||||
_CACHE.pop(canonical, None)
|
||||
key = _KEY_ALIASES.get(source, source)
|
||||
_CACHE.pop(key, None)
|
||||
# Also clear the source itself if it differed from key (alias case).
|
||||
if key != source:
|
||||
_CACHE.pop(source, None)
|
||||
logger.info("scraper_settings: cache cleared for source=%s", source)
|
||||
|
|
|
|||
|
|
@ -15,6 +15,7 @@ from typing import Any
|
|||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper
|
||||
from app.services.scrapers.yandex_helpers import (
|
||||
RE_AGENCY_FOUNDED,
|
||||
|
|
@ -103,7 +104,11 @@ class YandexDetailScraper(BaseScraper):
|
|||
|
||||
name = "yandex_detail"
|
||||
base_url = "https://realty.yandex.ru"
|
||||
request_delay_sec = 4.0
|
||||
request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
|
||||
# BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError
|
||||
async def fetch_around(
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from typing import Any
|
|||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper
|
||||
from app.services.scrapers.yandex_helpers import (
|
||||
RE_JK_ID,
|
||||
|
|
@ -122,7 +123,11 @@ _WORD_NUM: dict[str, int] = {
|
|||
class YandexNewbuildingScraper(BaseScraper):
|
||||
name = "yandex_realty_nb"
|
||||
base_url = "https://realty.yandex.ru"
|
||||
request_delay_sec = 4.0
|
||||
request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
|
||||
async def fetch_around(
|
||||
self, lat: float, lon: float, radius_m: int = 1000
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from typing import Any
|
|||
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper, ScrapedLot
|
||||
from app.services.scrapers.yandex_helpers import (
|
||||
RE_FLOOR,
|
||||
|
|
@ -48,6 +49,8 @@ class YandexRealtyScraper(BaseScraper):
|
|||
def __init__(self, city: str = DEFAULT_CITY) -> None:
|
||||
super().__init__()
|
||||
self.city = city
|
||||
# Load global Yandex delay from DB at instantiation
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
|
||||
async def fetch_around(
|
||||
self,
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from urllib.parse import urlencode
|
|||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser
|
||||
|
||||
from app.services.scraper_settings import get_scraper_delay
|
||||
from app.services.scrapers.base import BaseScraper
|
||||
from app.services.scrapers.yandex_helpers import (
|
||||
parse_dmy,
|
||||
|
|
@ -119,7 +120,11 @@ class YandexValuationScraper(BaseScraper):
|
|||
name = "yandex_valuation"
|
||||
base_url = "https://realty.yandex.ru"
|
||||
valuation_path = "/otsenka-kvartiry-po-adresu-onlayn/"
|
||||
request_delay_sec = 4.0
|
||||
request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
|
||||
async def fetch_around(
|
||||
self, lat: float, lon: float, radius_m: int = 1000
|
||||
|
|
|
|||
|
|
@ -1,19 +1,28 @@
|
|||
-- 053_scraper_settings.sql
|
||||
-- Таблица настроек парсеров: задержки между запросами (anti-ban), extensible.
|
||||
-- Одна строка на источник (source); UNIQUE(source).
|
||||
-- Purpose: Per-source live-config for scraper request delays (anti-ban knob).
|
||||
-- Loaded by app at runtime; UI can update without restart.
|
||||
-- Dependencies: none (standalone settings table)
|
||||
-- Sources: User request 2026-05-23 -- Yandex admin page + global delay knob.
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scraper_settings (
|
||||
id SERIAL PRIMARY KEY,
|
||||
source TEXT NOT NULL,
|
||||
request_delay_sec NUMERIC(6,2) NOT NULL DEFAULT 5.0,
|
||||
description TEXT,
|
||||
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
CONSTRAINT scraper_settings_source_uq UNIQUE (source)
|
||||
source text PRIMARY KEY, -- 'yandex' / 'avito' / 'cian' / 'n1' / 'yandex_detail' / ...
|
||||
request_delay_sec numeric(5,2) NOT NULL, -- seconds between requests (anti-ban). Range 1-60.
|
||||
updated_at timestamptz NOT NULL DEFAULT NOW(),
|
||||
-- Hint about who owns this knob (UI displays, no behavior)
|
||||
description text
|
||||
);
|
||||
|
||||
COMMENT ON TABLE scraper_settings IS
|
||||
'Per-source scraper delay settings. Special source=''global'' acts as floor across all scrapers.';
|
||||
'Per-source runtime config for scrapers. Currently exposes request_delay_sec '
|
||||
'(anti-ban delay between HTTP requests). Editable via admin UI; backend reads '
|
||||
'with 60s in-process cache. Source key matches BaseScraper.name in code.';
|
||||
|
||||
-- Seed: Yandex umbrella key applies to all 4 Yandex scrapers
|
||||
-- (yandex / yandex_detail / yandex_newbuilding / yandex_valuation read this key).
|
||||
INSERT INTO scraper_settings (source, request_delay_sec, description)
|
||||
VALUES ('yandex', 5.0, 'Global delay applied to all Yandex Realty scrapers (SERP, detail, newbuilding, valuation)')
|
||||
ON CONFLICT (source) DO NOTHING;
|
||||
|
||||
COMMIT;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
"""Offline tests for scraper_settings service (no DB required).
|
||||
"""Tests for scraper_settings loader (cache + DB fallback + alias mapping + global delay).
|
||||
|
||||
Тестируем логику max(per_source, global) через монкей-патч _get_setting_cached.
|
||||
Covers:
|
||||
- Per-source value lookup + caching (main's tests)
|
||||
- Global delay max() logic (our additions)
|
||||
- Admin endpoint smoke
|
||||
"""
|
||||
|
||||
import os
|
||||
|
|
@ -12,15 +15,49 @@ from unittest.mock import MagicMock, patch
|
|||
import pytest
|
||||
|
||||
import app.services.scraper_settings as ss_mod
|
||||
from app.services import scraper_settings as ss
|
||||
from app.services.scraper_settings import get_scraper_delay, invalidate_cache
|
||||
|
||||
# Patch target: _open_session is a module-level wrapper so tests can mock DB
|
||||
_PATCH_TARGET = "app.services.scraper_settings._open_session"
|
||||
|
||||
# ── helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
# ── Fixtures ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _reset_cache():
|
||||
ss._CACHE.clear()
|
||||
yield
|
||||
ss._CACHE.clear()
|
||||
|
||||
|
||||
# ── Helpers ───────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _mock_session_uniform(row_value: float | None) -> tuple[MagicMock, MagicMock]:
|
||||
"""Mock that returns the same value for any key (context-manager style).
|
||||
|
||||
Used by tests that don't care about multi-key distinction.
|
||||
Global key will also return the same value, so max() returns row_value.
|
||||
"""
|
||||
session = MagicMock()
|
||||
session.__enter__ = MagicMock(return_value=session)
|
||||
session.__exit__ = MagicMock(return_value=False)
|
||||
if row_value is None:
|
||||
session.execute.return_value.first.return_value = None
|
||||
else:
|
||||
session.execute.return_value.first.return_value = (row_value,)
|
||||
mock_open = MagicMock(return_value=session)
|
||||
return mock_open, session
|
||||
|
||||
|
||||
def _make_mock_session(values: dict[str, float | None]) -> MagicMock:
|
||||
"""Создать mock сессии БД с фиксированными значениями per source key."""
|
||||
"""Mock сессии БД с фиксированными значениями per source key.
|
||||
|
||||
Поддерживает разные значения для разных ключей — используется в тестах
|
||||
global delay логики.
|
||||
"""
|
||||
mock_db = MagicMock()
|
||||
|
||||
def fake_execute(sql, params):
|
||||
|
|
@ -36,11 +73,112 @@ def _make_mock_session(values: dict[str, float | None]) -> MagicMock:
|
|||
return mock_db
|
||||
|
||||
|
||||
# ── tests ─────────────────────────────────────────────────────────────────────
|
||||
# ── Main's tests: core cache + alias + fallback behaviour ─────────────────────
|
||||
|
||||
|
||||
def test_returns_db_value_for_yandex():
|
||||
"""DB value 7.5, global also 7.5 → max(7.5, 7.5) = 7.5."""
|
||||
mock_open, _ = _mock_session_uniform(7.5)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
assert ss.get_scraper_delay("yandex") == 7.5
|
||||
|
||||
|
||||
def test_yandex_subkeys_map_to_yandex_umbrella():
|
||||
"""yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row."""
|
||||
mock_open, _session = _mock_session_uniform(8.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
assert ss.get_scraper_delay("yandex_detail") == 8.0
|
||||
assert ss.get_scraper_delay("yandex_newbuilding") == 8.0
|
||||
assert ss.get_scraper_delay("yandex_valuation") == 8.0
|
||||
# First call fetches yandex + global (2 DB calls). Remaining 2 subkey calls
|
||||
# map to "yandex" (already cached) and "global" (already cached) → 0 more.
|
||||
assert mock_open.call_count == 2
|
||||
|
||||
|
||||
def test_cache_hit_within_ttl():
|
||||
"""After first call both yandex+global are cached; repeats hit cache only."""
|
||||
mock_open, _session = _mock_session_uniform(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex")
|
||||
ss.get_scraper_delay("yandex")
|
||||
ss.get_scraper_delay("yandex")
|
||||
# First call: 2 DB hits (yandex + global). Subsequent: 0.
|
||||
assert mock_open.call_count == 2
|
||||
|
||||
|
||||
def test_cache_expires_after_ttl():
|
||||
"""Force-age yandex key — next call re-fetches yandex only (global still fresh)."""
|
||||
mock_open, _session = _mock_session_uniform(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex") # +2 calls (yandex + global)
|
||||
# Force-age the yandex entry
|
||||
with ss._CACHE_LOCK:
|
||||
v, t = ss._CACHE["yandex"]
|
||||
ss._CACHE["yandex"] = (v, t - 999)
|
||||
ss.get_scraper_delay("yandex") # +1 call (yandex stale; global still fresh)
|
||||
assert mock_open.call_count == 3
|
||||
|
||||
|
||||
def test_db_missing_row_returns_default():
|
||||
"""Missing yandex row → _DEFAULT_DELAY_BY_SOURCE. Global None → 0.0. max = default."""
|
||||
mock_open, _ = _mock_session_uniform(None)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]
|
||||
|
||||
|
||||
def test_db_error_returns_default():
|
||||
"""DB error → hardcoded default. Global error → 0.0. max = default."""
|
||||
bad_session = MagicMock()
|
||||
bad_session.__enter__ = MagicMock(return_value=bad_session)
|
||||
bad_session.__exit__ = MagicMock(return_value=False)
|
||||
bad_session.execute.side_effect = RuntimeError("db down")
|
||||
bad_open = MagicMock(return_value=bad_session)
|
||||
with patch(_PATCH_TARGET, bad_open):
|
||||
assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]
|
||||
|
||||
|
||||
def test_unknown_source_uses_global_default():
|
||||
"""Unknown source, DB row None → _GLOBAL_DEFAULT_DELAY. Global None → 0.0. max = 5.0."""
|
||||
mock_open, _ = _mock_session_uniform(None)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY
|
||||
|
||||
|
||||
def test_invalidate_cache_specific_source():
|
||||
"""invalidate_cache(yandex) drops yandex entry; next call re-fetches yandex only."""
|
||||
mock_open, _ = _mock_session_uniform(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex") # +2 (yandex + global)
|
||||
ss.invalidate_cache("yandex") # clears yandex key
|
||||
ss.get_scraper_delay("yandex") # +1 (yandex stale; global still cached)
|
||||
assert mock_open.call_count == 3
|
||||
|
||||
|
||||
def test_invalidate_cache_all():
|
||||
"""invalidate_cache() clears all entries; next call re-fetches both."""
|
||||
mock_open, _ = _mock_session_uniform(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex") # +2 (yandex + global)
|
||||
ss.invalidate_cache() # clears everything
|
||||
ss.get_scraper_delay("yandex") # +2 (both stale)
|
||||
assert mock_open.call_count == 4
|
||||
|
||||
|
||||
def test_invalidate_alias_invalidates_umbrella():
|
||||
"""invalidate_cache('yandex_detail') clears the 'yandex' umbrella key."""
|
||||
mock_open, _ = _mock_session_uniform(6.0)
|
||||
with patch(_PATCH_TARGET, mock_open):
|
||||
ss.get_scraper_delay("yandex_detail") # +2 (yandex + global)
|
||||
ss.invalidate_cache("yandex_detail") # clears yandex (via alias)
|
||||
ss.get_scraper_delay("yandex_detail") # +1 (yandex stale; global fresh)
|
||||
assert mock_open.call_count == 3
|
||||
|
||||
|
||||
# ── Our additions: global delay max() logic ───────────────────────────────────
|
||||
|
||||
|
||||
def test_global_delay_takes_precedence_when_higher(monkeypatch):
|
||||
"""global=10 > avito=7 → возвращает 10."""
|
||||
"""global=10 > avito=7 → returns 10."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"avito": 7.0, "global": 10.0})
|
||||
|
|
@ -51,7 +189,7 @@ def test_global_delay_takes_precedence_when_higher(monkeypatch):
|
|||
|
||||
|
||||
def test_per_source_takes_precedence_when_higher(monkeypatch):
|
||||
"""avito=7 > global=2 → возвращает 7."""
|
||||
"""avito=7 > global=2 → returns 7."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"avito": 7.0, "global": 2.0})
|
||||
|
|
@ -62,7 +200,7 @@ def test_per_source_takes_precedence_when_higher(monkeypatch):
|
|||
|
||||
|
||||
def test_global_zero_means_no_floor(monkeypatch):
|
||||
"""global=0 → используется только per-source значение."""
|
||||
"""global=0 → only per-source value applies."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"cian": 5.0, "global": 0.0})
|
||||
|
|
@ -73,80 +211,18 @@ def test_global_zero_means_no_floor(monkeypatch):
|
|||
|
||||
|
||||
def test_missing_per_source_row_uses_class_default(monkeypatch):
|
||||
"""Если строки source в БД нет — фолбек на _DEFAULT_DELAY_BY_SOURCE."""
|
||||
"""Missing source row → _DEFAULT_DELAY_BY_SOURCE. global=0 → no floor."""
|
||||
invalidate_cache()
|
||||
|
||||
# n1 строки нет в БД, global=0
|
||||
mock_db = _make_mock_session({"n1": None, "global": 0.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
delay = get_scraper_delay("n1")
|
||||
# Дефолт для n1 = 5.0
|
||||
assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["n1"]
|
||||
|
||||
|
||||
def test_unknown_source_uses_global_default(monkeypatch):
|
||||
"""Неизвестный source без строки в БД — _GLOBAL_DEFAULT_DELAY."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"some_new_source": None, "global": 0.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
delay = get_scraper_delay("some_new_source")
|
||||
assert delay == ss_mod._GLOBAL_DEFAULT_DELAY
|
||||
|
||||
|
||||
def test_db_error_falls_back_to_default(monkeypatch):
|
||||
"""Ошибка БД → graceful fallback на дефолт, не crash."""
|
||||
invalidate_cache()
|
||||
|
||||
def broken_open():
|
||||
raise RuntimeError("DB is down")
|
||||
|
||||
monkeypatch.setattr(ss_mod, "_open_session", broken_open)
|
||||
|
||||
# Не должно кидать исключение
|
||||
delay = get_scraper_delay("avito")
|
||||
# При ошибке: per_source → _DEFAULT_DELAY_BY_SOURCE['avito'] = 7.0, global → 0.0
|
||||
assert delay == 7.0
|
||||
|
||||
|
||||
def test_invalidate_cache_single_source(monkeypatch):
|
||||
"""invalidate_cache(source) удаляет только указанный source."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
# Прогреть кеш для cian и avito
|
||||
get_scraper_delay("cian")
|
||||
get_scraper_delay("avito")
|
||||
|
||||
# Сбросить только cian
|
||||
invalidate_cache("cian")
|
||||
|
||||
with ss_mod._CACHE_LOCK:
|
||||
assert "cian" not in ss_mod._CACHE
|
||||
assert "avito" in ss_mod._CACHE
|
||||
|
||||
|
||||
def test_invalidate_cache_all(monkeypatch):
|
||||
"""invalidate_cache() без аргумента очищает весь кеш."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"avito": 7.0, "global": 0.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
get_scraper_delay("avito")
|
||||
|
||||
invalidate_cache()
|
||||
|
||||
with ss_mod._CACHE_LOCK:
|
||||
assert len(ss_mod._CACHE) == 0
|
||||
|
||||
|
||||
def test_global_missing_row_treated_as_zero(monkeypatch):
|
||||
"""Если строки 'global' нет в БД — фолбек 0.0 (не применяем floor)."""
|
||||
"""Missing 'global' row → fallback 0.0, avito=7.0 → max(7.0, 0.0) = 7.0."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"avito": 7.0, "global": None})
|
||||
|
|
@ -156,23 +232,66 @@ def test_global_missing_row_treated_as_zero(monkeypatch):
|
|||
assert delay == 7.0
|
||||
|
||||
|
||||
# ── Admin API integration smoke ───────────────────────────────────────────────
|
||||
def test_db_error_falls_back_to_default_global_logic(monkeypatch):
|
||||
"""DB error → graceful fallback, no crash. avito default=7.0, global error→0.0."""
|
||||
invalidate_cache()
|
||||
|
||||
def broken_open():
|
||||
raise RuntimeError("DB is down")
|
||||
|
||||
monkeypatch.setattr(ss_mod, "_open_session", broken_open)
|
||||
|
||||
delay = get_scraper_delay("avito")
|
||||
assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["avito"]
|
||||
|
||||
|
||||
def test_invalidate_cache_single_source_global_logic(monkeypatch):
|
||||
"""invalidate_cache(source) removes only that source from cache."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
get_scraper_delay("cian")
|
||||
get_scraper_delay("avito")
|
||||
|
||||
invalidate_cache("cian")
|
||||
|
||||
with ss_mod._CACHE_LOCK:
|
||||
assert "cian" not in ss_mod._CACHE
|
||||
assert "avito" in ss_mod._CACHE
|
||||
|
||||
|
||||
def test_invalidate_cache_all_global_logic(monkeypatch):
|
||||
"""invalidate_cache() without args clears the entire cache."""
|
||||
invalidate_cache()
|
||||
|
||||
mock_db = _make_mock_session({"avito": 7.0, "global": 0.0})
|
||||
monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
|
||||
|
||||
get_scraper_delay("avito")
|
||||
invalidate_cache()
|
||||
|
||||
with ss_mod._CACHE_LOCK:
|
||||
assert len(ss_mod._CACHE) == 0
|
||||
|
||||
|
||||
# ── Admin API endpoint smoke ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
def test_list_scraper_settings_endpoint(monkeypatch):
|
||||
"""GET /scraper-settings — эндпоинт монтируется, возвращает список."""
|
||||
"""GET /scraper-settings mounts and returns a list including 'global'."""
|
||||
import sys
|
||||
from unittest.mock import MagicMock as MM
|
||||
|
||||
# Stub weasyprint (нет на CI)
|
||||
_wp = MM()
|
||||
# Stub weasyprint (not on CI)
|
||||
_wp = MagicMock()
|
||||
sys.modules.setdefault("weasyprint", _wp)
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from app.api.v1 import admin as admin_module
|
||||
from app.core.db import get_db
|
||||
from fastapi import FastAPI
|
||||
|
||||
app = FastAPI()
|
||||
app.include_router(admin_module.router, prefix="/api/v1/admin")
|
||||
|
|
@ -182,7 +301,7 @@ def test_list_scraper_settings_endpoint(monkeypatch):
|
|||
{"source": "global", "request_delay_sec": 0.0, "description": "Global", "updated_at": None},
|
||||
]
|
||||
|
||||
mock_db = MM()
|
||||
mock_db = MagicMock()
|
||||
mock_db.execute.return_value.mappings.return_value.all.return_value = fake_rows
|
||||
app.dependency_overrides[get_db] = lambda: mock_db
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,35 @@
|
|||
"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings."""
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.scrapers.yandex_detail import YandexDetailScraper
|
||||
from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
|
||||
from app.services.scrapers.yandex_realty import YandexRealtyScraper
|
||||
from app.services.scrapers.yandex_valuation import YandexValuationScraper
|
||||
|
||||
|
||||
def test_yandex_realty_picks_up_delay():
|
||||
with patch("app.services.scrapers.yandex_realty.get_scraper_delay", return_value=9.5):
|
||||
s = YandexRealtyScraper()
|
||||
assert s.request_delay_sec == 9.5
|
||||
|
||||
|
||||
def test_yandex_detail_picks_up_delay():
|
||||
with patch("app.services.scrapers.yandex_detail.get_scraper_delay", return_value=9.5):
|
||||
s = YandexDetailScraper()
|
||||
assert s.request_delay_sec == 9.5
|
||||
|
||||
|
||||
def test_yandex_newbuilding_picks_up_delay():
|
||||
with patch(
|
||||
"app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5
|
||||
):
|
||||
s = YandexNewbuildingScraper()
|
||||
assert s.request_delay_sec == 9.5
|
||||
|
||||
|
||||
def test_yandex_valuation_picks_up_delay():
|
||||
with patch(
|
||||
"app.services.scrapers.yandex_valuation.get_scraper_delay", return_value=9.5
|
||||
):
|
||||
s = YandexValuationScraper()
|
||||
assert s.request_delay_sec == 9.5
|
||||
Loading…
Add table
Reference in a new issue