diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index 3f767eb1..d895abd0 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -35,7 +35,10 @@ from app.services.scrapers.avito_imv import ( from app.services.scrapers.base import save_listings from app.services.scrapers.cian import CianScraper from app.services.scrapers.n1 import N1Scraper +from app.services.scrapers.yandex_detail import YandexDetailScraper +from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper from app.services.scrapers.yandex_realty import YandexRealtyScraper +from app.services.scrapers.yandex_valuation import YandexValuationScraper logger = logging.getLogger(__name__) @@ -701,3 +704,122 @@ def update_schedule( next_run_at=row["next_run_at"].isoformat() if row["next_run_at"] else None, updated_at=row["updated_at"].isoformat() if row["updated_at"] else None, ) + + +# -- Yandex ad-hoc scrape triggers -------------------------------------------- + + +class YandexDetailTriggerResp(BaseModel): + ok: bool + offer_url: str + offer_id: str | None + price_rub: int | None + title: str | None + photo_count: int + + +@router.post("/scrape/yandex-detail", response_model=YandexDetailTriggerResp) +async def scrape_yandex_detail( + offer_url: str, +) -> YandexDetailTriggerResp: + """Ad-hoc parse one Yandex offer detail page. + + Returns a snapshot of the extracted DetailEnrichment (no DB write - read-only + debug; main pipeline writes via estimator on /estimate flow). + """ + async with YandexDetailScraper() as scraper: + result = await scraper.fetch_detail(offer_url) + if result is None: + raise HTTPException(404, f"Could not parse Yandex detail page: {offer_url}") + return YandexDetailTriggerResp( + ok=True, + offer_url=offer_url, + offer_id=result.offer_id, + price_rub=result.price_rub, + title=result.title, + photo_count=len(result.photo_urls), + ) + + +class YandexNewbuildingTriggerResp(BaseModel): + ok: bool + ext_id: str + ext_slug: str + name: str | None + lat: float | None + lon: float | None + rating: float | None + ratings_count: int | None + text_reviews_count: int | None + developer_name: str | None + + +@router.post("/scrape/yandex-newbuilding", response_model=YandexNewbuildingTriggerResp) +async def scrape_yandex_newbuilding( + slug: str, + id: str, + city: str = "ekaterinburg", +) -> YandexNewbuildingTriggerResp: + """Ad-hoc parse a Yandex JK landing page. + + URL: /{city}/kupit/novostrojka/-/ + """ + async with YandexNewbuildingScraper() as scraper: + result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city) + if result is None: + raise HTTPException(404, f"Could not parse Yandex JK: {slug}-{id} in {city}") + return YandexNewbuildingTriggerResp( + ok=True, + ext_id=result.ext_id, + ext_slug=result.ext_slug, + name=result.name, + lat=result.lat, + lon=result.lon, + rating=result.rating, + ratings_count=result.ratings_count, + text_reviews_count=result.text_reviews_count, + developer_name=result.developer_name, + ) + + +class YandexValuationTriggerResp(BaseModel): + ok: bool + address: str + year_built: int | None + total_floors: int | None + house_type: str | None + has_lift: bool | None + total_objects: int | None + history_items_count: int + + +@router.post("/scrape/yandex-valuation", response_model=YandexValuationTriggerResp) +async def scrape_yandex_valuation( + address: str, + offer_category: str = "APARTMENT", + offer_type: str = "SELL", + page: int = 1, +) -> YandexValuationTriggerResp: + """Ad-hoc fetch Yandex Valuation house-history for an address (debug, no cache write). + + Anonymous GET - works without auth. + """ + async with YandexValuationScraper() as scraper: + result = await scraper.fetch_house_history( + address=address, + offer_category=offer_category, + offer_type=offer_type, + page=page, + ) + if result is None: + raise HTTPException(404, f"Could not parse Yandex Valuation: {address}") + return YandexValuationTriggerResp( + ok=True, + address=address, + year_built=result.house.year_built, + total_floors=result.house.total_floors, + house_type=result.house.house_type, + has_lift=result.house.has_lift, + total_objects=result.house.total_objects, + history_items_count=len(result.history_items), + ) diff --git a/tradein-mvp/backend/app/services/scraper_settings.py b/tradein-mvp/backend/app/services/scraper_settings.py index 6f1bbf8a..cce374f6 100644 --- a/tradein-mvp/backend/app/services/scraper_settings.py +++ b/tradein-mvp/backend/app/services/scraper_settings.py @@ -12,60 +12,66 @@ invalidate_cache() вызывается из admin API PUT для немедле from __future__ import annotations import logging -import threading import time -from collections.abc import Generator -from contextlib import contextmanager +from threading import Lock from sqlalchemy import text -from sqlalchemy.orm import Session - -from app.core.db import SessionLocal logger = logging.getLogger(__name__) # Специальный ключ глобальной задержки (строка в scraper_settings с этим source). _GLOBAL_KEY = "global" -# TTL кеша в секундах — после истечения перечитывается из БД. +# In-process cache: {source: (delay_sec, fetched_at_epoch)} +_CACHE: dict[str, tuple[float, float]] = {} +_CACHE_LOCK = Lock() _CACHE_TTL_SEC = 60.0 -# Дефолтные задержки если строка в БД отсутствует. +# Default fallback if DB row missing or DB unreachable _DEFAULT_DELAY_BY_SOURCE: dict[str, float] = { "avito": 7.0, "cian": 5.0, "n1": 5.0, "yandex": 5.0, + "yandex_detail": 5.0, + "yandex_newbuilding": 5.0, + "yandex_valuation": 5.0, "domrf": 5.0, "rosreestr": 5.0, } - -# Fallback для неизвестного источника. _GLOBAL_DEFAULT_DELAY = 5.0 -# Алиасы источников: ключ → canonical source (можно расширять без изменения вызывающего кода). -_KEY_ALIASES: dict[str, str] = {} - -# Кеш: source → (delay_value, timestamp). -_CACHE: dict[str, tuple[float, float]] = {} -_CACHE_LOCK = threading.Lock() +# Yandex sub-scrapers all read the umbrella "yandex" key. +# Map: scraper_name -> DB row key. +_KEY_ALIASES: dict[str, str] = { + "yandex": "yandex", + "yandex_detail": "yandex", + "yandex_newbuilding": "yandex", + "yandex_realty_nb": "yandex", # BaseScraper.name used by YandexNewbuildingScraper + "yandex_valuation": "yandex", + # Non-Yandex sources resolve to their own keys for forward-compat + # (no rows exist yet for them; fall back to default). +} -@contextmanager -def _open_session() -> Generator[Session, None, None]: - """Открыть сессию БД с гарантированным закрытием.""" - db = SessionLocal() - try: - yield db - finally: - db.close() +def _open_session() -> object: + """Return a new SessionLocal context-manager instance. + + Deferred import avoids triggering Settings() validation at module import + time (unit tests run without DATABASE_URL env var). + Extracted as a named function so tests can patch it. + """ + from app.core.db import SessionLocal + + return SessionLocal() def _get_setting_cached(key: str) -> float: - """Кеш-aware чтение одного source из scraper_settings. + """Cache-aware read of one source key from scraper_settings. - Возвращает request_delay_sec из БД (с кешем TTL=60s). - При отсутствии строки — фолбек на _DEFAULT_DELAY_BY_SOURCE или 0.0 для global. + Returns request_delay_sec from DB (with TTL=60s cache). + Missing row: falls back to _DEFAULT_DELAY_BY_SOURCE, or 0.0 for _GLOBAL_KEY + (meaning no global floor is applied when the row hasn't been seeded yet). """ now = time.time() with _CACHE_LOCK: @@ -76,20 +82,20 @@ def _get_setting_cached(key: str) -> float: try: with _open_session() as db: row = db.execute( - text("SELECT request_delay_sec FROM scraper_settings WHERE source = :s"), + text( + "SELECT request_delay_sec FROM scraper_settings WHERE source = :s" + ), {"s": key}, ).first() if row is not None: value = float(row[0]) elif key == _GLOBAL_KEY: - # Глобальная строка ещё не создана — не применяем floor. + # Global row not yet seeded — don't apply floor. value = 0.0 else: value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY) - except Exception as exc: - logger.warning( - "scraper_settings: load failed for %s — using default: %s", key, exc - ) + except Exception as e: + logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e) value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get( key, _GLOBAL_DEFAULT_DELAY ) @@ -100,10 +106,13 @@ def _get_setting_cached(key: str) -> float: def get_scraper_delay(source: str) -> float: - """Вернуть эффективную задержку для source = max(per_source, global). + """Return the configured request_delay_sec for `source`. - Строка source='global' задаёт нижнюю планку для всех парсеров. - Если global=0 — используется только per-source значение. + Effective delay = max(per_source, global). + - Maps Yandex sub-scrapers to the umbrella 'yandex' key. + - Cached 60s in-process to avoid per-instance DB query. + - source='global' sets a floor across all scrapers (0 = disabled). + - Any DB error -> returns the hardcoded default for that source. """ key = _KEY_ALIASES.get(source, source) per_source = _get_setting_cached(key) @@ -112,19 +121,18 @@ def get_scraper_delay(source: str) -> float: def invalidate_cache(source: str | None = None) -> None: - """Сбросить кеш для source (или весь кеш если source=None). + """Clear the cache entry for `source`, or the whole cache if None. - Вызывается из admin API PUT /scraper-settings/{source} для немедленного - применения нового значения без ожидания TTL. + Called by admin endpoint after PUT to force a re-read on next get_scraper_delay(). """ with _CACHE_LOCK: if source is None: _CACHE.clear() logger.info("scraper_settings: cache cleared (all sources)") else: - _CACHE.pop(source, None) - # Если source — алиас, сбрасываем canonical key тоже. - canonical = _KEY_ALIASES.get(source) - if canonical: - _CACHE.pop(canonical, None) + key = _KEY_ALIASES.get(source, source) + _CACHE.pop(key, None) + # Also clear the source itself if it differed from key (alias case). + if key != source: + _CACHE.pop(source, None) logger.info("scraper_settings: cache cleared for source=%s", source) diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_detail.py b/tradein-mvp/backend/app/services/scrapers/yandex_detail.py index 4a4985ec..0d4c6101 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_detail.py @@ -15,6 +15,7 @@ from typing import Any from pydantic import BaseModel, Field from selectolax.parser import HTMLParser, Node +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper from app.services.scrapers.yandex_helpers import ( RE_AGENCY_FOUNDED, @@ -103,7 +104,11 @@ class YandexDetailScraper(BaseScraper): name = "yandex_detail" base_url = "https://realty.yandex.ru" - request_delay_sec = 4.0 + request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings + + def __init__(self) -> None: + super().__init__() + self.request_delay_sec = get_scraper_delay(self.name) # BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError async def fetch_around( diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py b/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py index cdca4db5..9139e77a 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py @@ -14,6 +14,7 @@ from typing import Any from pydantic import BaseModel, Field from selectolax.parser import HTMLParser, Node +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper from app.services.scrapers.yandex_helpers import ( RE_JK_ID, @@ -122,7 +123,11 @@ _WORD_NUM: dict[str, int] = { class YandexNewbuildingScraper(BaseScraper): name = "yandex_realty_nb" base_url = "https://realty.yandex.ru" - request_delay_sec = 4.0 + request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings + + def __init__(self) -> None: + super().__init__() + self.request_delay_sec = get_scraper_delay(self.name) async def fetch_around( self, lat: float, lon: float, radius_m: int = 1000 diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py index 4f0cf654..cae46c71 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py @@ -18,6 +18,7 @@ from typing import Any from selectolax.parser import HTMLParser, Node +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot from app.services.scrapers.yandex_helpers import ( RE_FLOOR, @@ -48,6 +49,8 @@ class YandexRealtyScraper(BaseScraper): def __init__(self, city: str = DEFAULT_CITY) -> None: super().__init__() self.city = city + # Load global Yandex delay from DB at instantiation + self.request_delay_sec = get_scraper_delay(self.name) async def fetch_around( self, diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py index daa3606c..425a01f0 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py @@ -25,6 +25,7 @@ from urllib.parse import urlencode from pydantic import BaseModel, Field from selectolax.parser import HTMLParser +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper from app.services.scrapers.yandex_helpers import ( parse_dmy, @@ -119,7 +120,11 @@ class YandexValuationScraper(BaseScraper): name = "yandex_valuation" base_url = "https://realty.yandex.ru" valuation_path = "/otsenka-kvartiry-po-adresu-onlayn/" - request_delay_sec = 4.0 + request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings + + def __init__(self) -> None: + super().__init__() + self.request_delay_sec = get_scraper_delay(self.name) async def fetch_around( self, lat: float, lon: float, radius_m: int = 1000 diff --git a/tradein-mvp/backend/data/sql/053_scraper_settings.sql b/tradein-mvp/backend/data/sql/053_scraper_settings.sql index 71fd4bef..c65db0cf 100644 --- a/tradein-mvp/backend/data/sql/053_scraper_settings.sql +++ b/tradein-mvp/backend/data/sql/053_scraper_settings.sql @@ -1,19 +1,28 @@ -- 053_scraper_settings.sql --- Таблица настроек парсеров: задержки между запросами (anti-ban), extensible. --- Одна строка на источник (source); UNIQUE(source). +-- Purpose: Per-source live-config for scraper request delays (anti-ban knob). +-- Loaded by app at runtime; UI can update without restart. +-- Dependencies: none (standalone settings table) +-- Sources: User request 2026-05-23 -- Yandex admin page + global delay knob. + BEGIN; CREATE TABLE IF NOT EXISTS scraper_settings ( - id SERIAL PRIMARY KEY, - source TEXT NOT NULL, - request_delay_sec NUMERIC(6,2) NOT NULL DEFAULT 5.0, - description TEXT, - created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), - CONSTRAINT scraper_settings_source_uq UNIQUE (source) + source text PRIMARY KEY, -- 'yandex' / 'avito' / 'cian' / 'n1' / 'yandex_detail' / ... + request_delay_sec numeric(5,2) NOT NULL, -- seconds between requests (anti-ban). Range 1-60. + updated_at timestamptz NOT NULL DEFAULT NOW(), + -- Hint about who owns this knob (UI displays, no behavior) + description text ); COMMENT ON TABLE scraper_settings IS - 'Per-source scraper delay settings. Special source=''global'' acts as floor across all scrapers.'; + 'Per-source runtime config for scrapers. Currently exposes request_delay_sec ' + '(anti-ban delay between HTTP requests). Editable via admin UI; backend reads ' + 'with 60s in-process cache. Source key matches BaseScraper.name in code.'; + +-- Seed: Yandex umbrella key applies to all 4 Yandex scrapers +-- (yandex / yandex_detail / yandex_newbuilding / yandex_valuation read this key). +INSERT INTO scraper_settings (source, request_delay_sec, description) +VALUES ('yandex', 5.0, 'Global delay applied to all Yandex Realty scrapers (SERP, detail, newbuilding, valuation)') +ON CONFLICT (source) DO NOTHING; COMMIT; diff --git a/tradein-mvp/backend/tests/test_scraper_settings.py b/tradein-mvp/backend/tests/test_scraper_settings.py index e754e2ad..a465382c 100644 --- a/tradein-mvp/backend/tests/test_scraper_settings.py +++ b/tradein-mvp/backend/tests/test_scraper_settings.py @@ -1,6 +1,9 @@ -"""Offline tests for scraper_settings service (no DB required). +"""Tests for scraper_settings loader (cache + DB fallback + alias mapping + global delay). -Тестируем логику max(per_source, global) через монкей-патч _get_setting_cached. +Covers: + - Per-source value lookup + caching (main's tests) + - Global delay max() logic (our additions) + - Admin endpoint smoke """ import os @@ -12,15 +15,49 @@ from unittest.mock import MagicMock, patch import pytest import app.services.scraper_settings as ss_mod +from app.services import scraper_settings as ss from app.services.scraper_settings import get_scraper_delay, invalidate_cache +# Patch target: _open_session is a module-level wrapper so tests can mock DB +_PATCH_TARGET = "app.services.scraper_settings._open_session" -# ── helpers ────────────────────────────────────────────────────────────────── + +# ── Fixtures ────────────────────────────────────────────────────────────────── + + +@pytest.fixture(autouse=True) +def _reset_cache(): + ss._CACHE.clear() + yield + ss._CACHE.clear() + + +# ── Helpers ─────────────────────────────────────────────────────────────────── + + +def _mock_session_uniform(row_value: float | None) -> tuple[MagicMock, MagicMock]: + """Mock that returns the same value for any key (context-manager style). + + Used by tests that don't care about multi-key distinction. + Global key will also return the same value, so max() returns row_value. + """ + session = MagicMock() + session.__enter__ = MagicMock(return_value=session) + session.__exit__ = MagicMock(return_value=False) + if row_value is None: + session.execute.return_value.first.return_value = None + else: + session.execute.return_value.first.return_value = (row_value,) + mock_open = MagicMock(return_value=session) + return mock_open, session def _make_mock_session(values: dict[str, float | None]) -> MagicMock: - """Создать mock сессии БД с фиксированными значениями per source key.""" + """Mock сессии БД с фиксированными значениями per source key. + Поддерживает разные значения для разных ключей — используется в тестах + global delay логики. + """ mock_db = MagicMock() def fake_execute(sql, params): @@ -36,11 +73,112 @@ def _make_mock_session(values: dict[str, float | None]) -> MagicMock: return mock_db -# ── tests ───────────────────────────────────────────────────────────────────── +# ── Main's tests: core cache + alias + fallback behaviour ───────────────────── + + +def test_returns_db_value_for_yandex(): + """DB value 7.5, global also 7.5 → max(7.5, 7.5) = 7.5.""" + mock_open, _ = _mock_session_uniform(7.5) + with patch(_PATCH_TARGET, mock_open): + assert ss.get_scraper_delay("yandex") == 7.5 + + +def test_yandex_subkeys_map_to_yandex_umbrella(): + """yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row.""" + mock_open, _session = _mock_session_uniform(8.0) + with patch(_PATCH_TARGET, mock_open): + assert ss.get_scraper_delay("yandex_detail") == 8.0 + assert ss.get_scraper_delay("yandex_newbuilding") == 8.0 + assert ss.get_scraper_delay("yandex_valuation") == 8.0 + # First call fetches yandex + global (2 DB calls). Remaining 2 subkey calls + # map to "yandex" (already cached) and "global" (already cached) → 0 more. + assert mock_open.call_count == 2 + + +def test_cache_hit_within_ttl(): + """After first call both yandex+global are cached; repeats hit cache only.""" + mock_open, _session = _mock_session_uniform(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex") + ss.get_scraper_delay("yandex") + ss.get_scraper_delay("yandex") + # First call: 2 DB hits (yandex + global). Subsequent: 0. + assert mock_open.call_count == 2 + + +def test_cache_expires_after_ttl(): + """Force-age yandex key — next call re-fetches yandex only (global still fresh).""" + mock_open, _session = _mock_session_uniform(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex") # +2 calls (yandex + global) + # Force-age the yandex entry + with ss._CACHE_LOCK: + v, t = ss._CACHE["yandex"] + ss._CACHE["yandex"] = (v, t - 999) + ss.get_scraper_delay("yandex") # +1 call (yandex stale; global still fresh) + assert mock_open.call_count == 3 + + +def test_db_missing_row_returns_default(): + """Missing yandex row → _DEFAULT_DELAY_BY_SOURCE. Global None → 0.0. max = default.""" + mock_open, _ = _mock_session_uniform(None) + with patch(_PATCH_TARGET, mock_open): + assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"] + + +def test_db_error_returns_default(): + """DB error → hardcoded default. Global error → 0.0. max = default.""" + bad_session = MagicMock() + bad_session.__enter__ = MagicMock(return_value=bad_session) + bad_session.__exit__ = MagicMock(return_value=False) + bad_session.execute.side_effect = RuntimeError("db down") + bad_open = MagicMock(return_value=bad_session) + with patch(_PATCH_TARGET, bad_open): + assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"] + + +def test_unknown_source_uses_global_default(): + """Unknown source, DB row None → _GLOBAL_DEFAULT_DELAY. Global None → 0.0. max = 5.0.""" + mock_open, _ = _mock_session_uniform(None) + with patch(_PATCH_TARGET, mock_open): + assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY + + +def test_invalidate_cache_specific_source(): + """invalidate_cache(yandex) drops yandex entry; next call re-fetches yandex only.""" + mock_open, _ = _mock_session_uniform(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex") # +2 (yandex + global) + ss.invalidate_cache("yandex") # clears yandex key + ss.get_scraper_delay("yandex") # +1 (yandex stale; global still cached) + assert mock_open.call_count == 3 + + +def test_invalidate_cache_all(): + """invalidate_cache() clears all entries; next call re-fetches both.""" + mock_open, _ = _mock_session_uniform(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex") # +2 (yandex + global) + ss.invalidate_cache() # clears everything + ss.get_scraper_delay("yandex") # +2 (both stale) + assert mock_open.call_count == 4 + + +def test_invalidate_alias_invalidates_umbrella(): + """invalidate_cache('yandex_detail') clears the 'yandex' umbrella key.""" + mock_open, _ = _mock_session_uniform(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex_detail") # +2 (yandex + global) + ss.invalidate_cache("yandex_detail") # clears yandex (via alias) + ss.get_scraper_delay("yandex_detail") # +1 (yandex stale; global fresh) + assert mock_open.call_count == 3 + + +# ── Our additions: global delay max() logic ─────────────────────────────────── def test_global_delay_takes_precedence_when_higher(monkeypatch): - """global=10 > avito=7 → возвращает 10.""" + """global=10 > avito=7 → returns 10.""" invalidate_cache() mock_db = _make_mock_session({"avito": 7.0, "global": 10.0}) @@ -51,7 +189,7 @@ def test_global_delay_takes_precedence_when_higher(monkeypatch): def test_per_source_takes_precedence_when_higher(monkeypatch): - """avito=7 > global=2 → возвращает 7.""" + """avito=7 > global=2 → returns 7.""" invalidate_cache() mock_db = _make_mock_session({"avito": 7.0, "global": 2.0}) @@ -62,7 +200,7 @@ def test_per_source_takes_precedence_when_higher(monkeypatch): def test_global_zero_means_no_floor(monkeypatch): - """global=0 → используется только per-source значение.""" + """global=0 → only per-source value applies.""" invalidate_cache() mock_db = _make_mock_session({"cian": 5.0, "global": 0.0}) @@ -73,80 +211,18 @@ def test_global_zero_means_no_floor(monkeypatch): def test_missing_per_source_row_uses_class_default(monkeypatch): - """Если строки source в БД нет — фолбек на _DEFAULT_DELAY_BY_SOURCE.""" + """Missing source row → _DEFAULT_DELAY_BY_SOURCE. global=0 → no floor.""" invalidate_cache() - # n1 строки нет в БД, global=0 mock_db = _make_mock_session({"n1": None, "global": 0.0}) monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) delay = get_scraper_delay("n1") - # Дефолт для n1 = 5.0 assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["n1"] -def test_unknown_source_uses_global_default(monkeypatch): - """Неизвестный source без строки в БД — _GLOBAL_DEFAULT_DELAY.""" - invalidate_cache() - - mock_db = _make_mock_session({"some_new_source": None, "global": 0.0}) - monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) - - delay = get_scraper_delay("some_new_source") - assert delay == ss_mod._GLOBAL_DEFAULT_DELAY - - -def test_db_error_falls_back_to_default(monkeypatch): - """Ошибка БД → graceful fallback на дефолт, не crash.""" - invalidate_cache() - - def broken_open(): - raise RuntimeError("DB is down") - - monkeypatch.setattr(ss_mod, "_open_session", broken_open) - - # Не должно кидать исключение - delay = get_scraper_delay("avito") - # При ошибке: per_source → _DEFAULT_DELAY_BY_SOURCE['avito'] = 7.0, global → 0.0 - assert delay == 7.0 - - -def test_invalidate_cache_single_source(monkeypatch): - """invalidate_cache(source) удаляет только указанный source.""" - invalidate_cache() - - mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0}) - monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) - - # Прогреть кеш для cian и avito - get_scraper_delay("cian") - get_scraper_delay("avito") - - # Сбросить только cian - invalidate_cache("cian") - - with ss_mod._CACHE_LOCK: - assert "cian" not in ss_mod._CACHE - assert "avito" in ss_mod._CACHE - - -def test_invalidate_cache_all(monkeypatch): - """invalidate_cache() без аргумента очищает весь кеш.""" - invalidate_cache() - - mock_db = _make_mock_session({"avito": 7.0, "global": 0.0}) - monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) - - get_scraper_delay("avito") - - invalidate_cache() - - with ss_mod._CACHE_LOCK: - assert len(ss_mod._CACHE) == 0 - - def test_global_missing_row_treated_as_zero(monkeypatch): - """Если строки 'global' нет в БД — фолбек 0.0 (не применяем floor).""" + """Missing 'global' row → fallback 0.0, avito=7.0 → max(7.0, 0.0) = 7.0.""" invalidate_cache() mock_db = _make_mock_session({"avito": 7.0, "global": None}) @@ -156,23 +232,66 @@ def test_global_missing_row_treated_as_zero(monkeypatch): assert delay == 7.0 -# ── Admin API integration smoke ─────────────────────────────────────────────── +def test_db_error_falls_back_to_default_global_logic(monkeypatch): + """DB error → graceful fallback, no crash. avito default=7.0, global error→0.0.""" + invalidate_cache() + + def broken_open(): + raise RuntimeError("DB is down") + + monkeypatch.setattr(ss_mod, "_open_session", broken_open) + + delay = get_scraper_delay("avito") + assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["avito"] + + +def test_invalidate_cache_single_source_global_logic(monkeypatch): + """invalidate_cache(source) removes only that source from cache.""" + invalidate_cache() + + mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0}) + monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) + + get_scraper_delay("cian") + get_scraper_delay("avito") + + invalidate_cache("cian") + + with ss_mod._CACHE_LOCK: + assert "cian" not in ss_mod._CACHE + assert "avito" in ss_mod._CACHE + + +def test_invalidate_cache_all_global_logic(monkeypatch): + """invalidate_cache() without args clears the entire cache.""" + invalidate_cache() + + mock_db = _make_mock_session({"avito": 7.0, "global": 0.0}) + monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db) + + get_scraper_delay("avito") + invalidate_cache() + + with ss_mod._CACHE_LOCK: + assert len(ss_mod._CACHE) == 0 + + +# ── Admin API endpoint smoke ─────────────────────────────────────────────────── def test_list_scraper_settings_endpoint(monkeypatch): - """GET /scraper-settings — эндпоинт монтируется, возвращает список.""" + """GET /scraper-settings mounts and returns a list including 'global'.""" import sys - from unittest.mock import MagicMock as MM - # Stub weasyprint (нет на CI) - _wp = MM() + # Stub weasyprint (not on CI) + _wp = MagicMock() sys.modules.setdefault("weasyprint", _wp) + from fastapi import FastAPI from fastapi.testclient import TestClient from app.api.v1 import admin as admin_module from app.core.db import get_db - from fastapi import FastAPI app = FastAPI() app.include_router(admin_module.router, prefix="/api/v1/admin") @@ -182,7 +301,7 @@ def test_list_scraper_settings_endpoint(monkeypatch): {"source": "global", "request_delay_sec": 0.0, "description": "Global", "updated_at": None}, ] - mock_db = MM() + mock_db = MagicMock() mock_db.execute.return_value.mappings.return_value.all.return_value = fake_rows app.dependency_overrides[get_db] = lambda: mock_db diff --git a/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py b/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py new file mode 100644 index 00000000..3b262ebc --- /dev/null +++ b/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py @@ -0,0 +1,35 @@ +"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings.""" +from unittest.mock import patch + +from app.services.scrapers.yandex_detail import YandexDetailScraper +from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper +from app.services.scrapers.yandex_realty import YandexRealtyScraper +from app.services.scrapers.yandex_valuation import YandexValuationScraper + + +def test_yandex_realty_picks_up_delay(): + with patch("app.services.scrapers.yandex_realty.get_scraper_delay", return_value=9.5): + s = YandexRealtyScraper() + assert s.request_delay_sec == 9.5 + + +def test_yandex_detail_picks_up_delay(): + with patch("app.services.scrapers.yandex_detail.get_scraper_delay", return_value=9.5): + s = YandexDetailScraper() + assert s.request_delay_sec == 9.5 + + +def test_yandex_newbuilding_picks_up_delay(): + with patch( + "app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5 + ): + s = YandexNewbuildingScraper() + assert s.request_delay_sec == 9.5 + + +def test_yandex_valuation_picks_up_delay(): + with patch( + "app.services.scrapers.yandex_valuation.get_scraper_delay", return_value=9.5 + ): + s = YandexValuationScraper() + assert s.request_delay_sec == 9.5