From e5fb209ced188d266179d4b20545e9c5d80e2faf Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 18:21:39 +0300 Subject: [PATCH] feat(tradein): scraper_settings -- live-config request delay for Yandex scrapers + admin triggers Backend foundation for the Yandex admin scraper page (UI follows in next PR). scraper_settings (mig 053): - New table source PK + request_delay_sec numeric(5,2) + updated_at + description - Seeded with row 'yandex' = 5.0s (umbrella key for all 4 Yandex scrapers) scraper_settings.py (new): - get_scraper_delay(source) -> float, with 60s in-process cache + DB fallback - Yandex sub-scrapers (yandex_detail / yandex_newbuilding / yandex_valuation) resolve to umbrella 'yandex' key via _KEY_ALIASES - invalidate_cache(source | None) for admin PUT side-effect - Deferred SessionLocal import (_open_session helper) for unit-test compat 4 Yandex scrapers wired to load delay at __init__: - yandex_realty.py / yandex_detail.py / yandex_newbuilding.py / yandex_valuation.py - Class default bumped to 5.0; instance value overridden via get_scraper_delay(self.name) admin.py -- 5 new endpoints: - GET /scraper-settings list[ScraperSetting] - PUT /scraper-settings/{source} upsert + invalidate_cache - POST /scrape/yandex-detail offer_url= -> detail snapshot - POST /scrape/yandex-newbuilding slug=, id=, city=ekaterinburg -> JK snapshot - POST /scrape/yandex-valuation address=, offer_category=, offer_type=, page= -> house meta Tests: 10 + 4 = 14 new unit tests (loader cache/alias/error paths + scraper wiring). Ruff clean. --- tradein-mvp/backend/app/api/v1/admin.py | 206 ++++++++++++++++++ .../backend/app/services/scraper_settings.py | 101 +++++++++ .../app/services/scrapers/yandex_detail.py | 7 +- .../services/scrapers/yandex_newbuilding.py | 7 +- .../app/services/scrapers/yandex_realty.py | 3 + .../app/services/scrapers/yandex_valuation.py | 7 +- .../backend/data/sql/053_scraper_settings.sql | 28 +++ .../backend/tests/test_scraper_settings.py | 118 ++++++++++ .../test_yandex_scrapers_delay_wiring.py | 35 +++ 9 files changed, 509 insertions(+), 3 deletions(-) create mode 100644 tradein-mvp/backend/app/services/scraper_settings.py create mode 100644 tradein-mvp/backend/data/sql/053_scraper_settings.sql create mode 100644 tradein-mvp/backend/tests/test_scraper_settings.py create mode 100644 tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index 9fa51729..2d29b569 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -22,6 +22,7 @@ from app.services import cian_session as cian_session_svc from app.services import scrape_runs as runs_mod from app.services.geocoder import geocode from app.services.scrape_pipeline import run_avito_city_sweep +from app.services.scraper_settings import invalidate_cache from app.services.scrapers.avito import AvitoScraper from app.services.scrapers.avito_detail import fetch_detail, save_detail_enrichment from app.services.scrapers.avito_houses import fetch_house_catalog, save_house_catalog_enrichment @@ -34,7 +35,10 @@ from app.services.scrapers.avito_imv import ( from app.services.scrapers.base import save_listings from app.services.scrapers.cian import CianScraper from app.services.scrapers.n1 import N1Scraper +from app.services.scrapers.yandex_detail import YandexDetailScraper +from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper from app.services.scrapers.yandex_realty import YandexRealtyScraper +from app.services.scrapers.yandex_valuation import YandexValuationScraper logger = logging.getLogger(__name__) @@ -75,6 +79,17 @@ class ScrapeResponse(BaseModel): by_source: list[ScrapeResult] +class ScraperSetting(BaseModel): + source: str + request_delay_sec: float + description: str | None = None + updated_at: str | None = None # ISO + + +class ScraperSettingUpdate(BaseModel): + request_delay_sec: float = Field(ge=1.0, le=60.0) + + @router.post("/scrape", response_model=ScrapeResponse) async def scrape_around( payload: ScrapeRequest, @@ -632,3 +647,194 @@ def update_schedule( next_run_at=row["next_run_at"].isoformat() if row["next_run_at"] else None, updated_at=row["updated_at"].isoformat() if row["updated_at"] else None, ) + + +# -- Scraper settings (per-source live config) -------------------------------- + + +@router.get("/scraper-settings", response_model=list[ScraperSetting]) +async def list_scraper_settings( + db: Annotated[Session, Depends(get_db)], +) -> list[ScraperSetting]: + """Return all scraper_settings rows (live config for delays).""" + rows = ( + db.execute( + text( + """ + SELECT source, request_delay_sec, description, updated_at + FROM scraper_settings + ORDER BY source + """ + ) + ) + .mappings() + .all() + ) + return [ + ScraperSetting( + source=r["source"], + request_delay_sec=float(r["request_delay_sec"]), + description=r["description"], + updated_at=r["updated_at"].isoformat() if r["updated_at"] else None, + ) + for r in rows + ] + + +@router.put("/scraper-settings/{source}", response_model=ScraperSetting) +async def update_scraper_setting( + source: str, + payload: ScraperSettingUpdate, + db: Annotated[Session, Depends(get_db)], +) -> ScraperSetting: + """Update request_delay_sec for one source. UPSERT - creates row if missing. + + Invalidates the in-process cache so the change takes effect on next + scraper instantiation (usually within seconds). + """ + row = ( + db.execute( + text( + """ + INSERT INTO scraper_settings (source, request_delay_sec, updated_at) + VALUES (:source, :delay, NOW()) + ON CONFLICT (source) DO UPDATE + SET request_delay_sec = EXCLUDED.request_delay_sec, + updated_at = NOW() + RETURNING source, request_delay_sec, description, updated_at + """ + ), + {"source": source, "delay": payload.request_delay_sec}, + ) + .mappings() + .first() + ) + db.commit() + invalidate_cache(source) + if row is None: + raise HTTPException(500, "Failed to upsert scraper_settings row") + return ScraperSetting( + source=row["source"], + request_delay_sec=float(row["request_delay_sec"]), + description=row["description"], + updated_at=row["updated_at"].isoformat() if row["updated_at"] else None, + ) + + +# -- Yandex ad-hoc scrape triggers -------------------------------------------- + + +class YandexDetailTriggerResp(BaseModel): + ok: bool + offer_url: str + offer_id: str | None + price_rub: int | None + title: str | None + photo_count: int + + +@router.post("/scrape/yandex-detail", response_model=YandexDetailTriggerResp) +async def scrape_yandex_detail( + offer_url: str, +) -> YandexDetailTriggerResp: + """Ad-hoc parse one Yandex offer detail page. + + Returns a snapshot of the extracted DetailEnrichment (no DB write - read-only + debug; main pipeline writes via estimator on /estimate flow). + """ + async with YandexDetailScraper() as scraper: + result = await scraper.fetch_detail(offer_url) + if result is None: + raise HTTPException(404, f"Could not parse Yandex detail page: {offer_url}") + return YandexDetailTriggerResp( + ok=True, + offer_url=offer_url, + offer_id=result.offer_id, + price_rub=result.price_rub, + title=result.title, + photo_count=len(result.photo_urls), + ) + + +class YandexNewbuildingTriggerResp(BaseModel): + ok: bool + ext_id: str + ext_slug: str + name: str | None + lat: float | None + lon: float | None + rating: float | None + ratings_count: int | None + text_reviews_count: int | None + developer_name: str | None + + +@router.post("/scrape/yandex-newbuilding", response_model=YandexNewbuildingTriggerResp) +async def scrape_yandex_newbuilding( + slug: str, + id: str, + city: str = "ekaterinburg", +) -> YandexNewbuildingTriggerResp: + """Ad-hoc parse a Yandex JK landing page. + + URL: /{city}/kupit/novostrojka/-/ + """ + async with YandexNewbuildingScraper() as scraper: + result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city) + if result is None: + raise HTTPException(404, f"Could not parse Yandex JK: {slug}-{id} in {city}") + return YandexNewbuildingTriggerResp( + ok=True, + ext_id=result.ext_id, + ext_slug=result.ext_slug, + name=result.name, + lat=result.lat, + lon=result.lon, + rating=result.rating, + ratings_count=result.ratings_count, + text_reviews_count=result.text_reviews_count, + developer_name=result.developer_name, + ) + + +class YandexValuationTriggerResp(BaseModel): + ok: bool + address: str + year_built: int | None + total_floors: int | None + house_type: str | None + has_lift: bool | None + total_objects: int | None + history_items_count: int + + +@router.post("/scrape/yandex-valuation", response_model=YandexValuationTriggerResp) +async def scrape_yandex_valuation( + address: str, + offer_category: str = "APARTMENT", + offer_type: str = "SELL", + page: int = 1, +) -> YandexValuationTriggerResp: + """Ad-hoc fetch Yandex Valuation house-history for an address (debug, no cache write). + + Anonymous GET - works without auth. + """ + async with YandexValuationScraper() as scraper: + result = await scraper.fetch_house_history( + address=address, + offer_category=offer_category, + offer_type=offer_type, + page=page, + ) + if result is None: + raise HTTPException(404, f"Could not parse Yandex Valuation: {address}") + return YandexValuationTriggerResp( + ok=True, + address=address, + year_built=result.house.year_built, + total_floors=result.house.total_floors, + house_type=result.house.house_type, + has_lift=result.house.has_lift, + total_objects=result.house.total_objects, + history_items_count=len(result.history_items), + ) diff --git a/tradein-mvp/backend/app/services/scraper_settings.py b/tradein-mvp/backend/app/services/scraper_settings.py new file mode 100644 index 00000000..26d0d33c --- /dev/null +++ b/tradein-mvp/backend/app/services/scraper_settings.py @@ -0,0 +1,101 @@ +"""Runtime config loader for scraper delays. + +Backed by scraper_settings table with 60s in-process cache. +""" + +from __future__ import annotations + +import logging +import time +from threading import Lock + +from sqlalchemy import text + +logger = logging.getLogger(__name__) + +# In-process cache: {source: (delay_sec, fetched_at_epoch)} +_CACHE: dict[str, tuple[float, float]] = {} +_CACHE_LOCK = Lock() +_CACHE_TTL_SEC = 60.0 + +# Default fallback if DB row missing or DB unreachable +_DEFAULT_DELAY_BY_SOURCE: dict[str, float] = { + "yandex": 5.0, + "yandex_detail": 5.0, + "yandex_newbuilding": 5.0, + "yandex_valuation": 5.0, +} +_GLOBAL_DEFAULT_DELAY = 5.0 + +# Yandex sub-scrapers all read the umbrella "yandex" key. +# Map: scraper_name -> DB row key. +_KEY_ALIASES: dict[str, str] = { + "yandex": "yandex", + "yandex_detail": "yandex", + "yandex_newbuilding": "yandex", + "yandex_realty_nb": "yandex", # BaseScraper.name used by YandexNewbuildingScraper + "yandex_valuation": "yandex", + # Non-Yandex sources resolve to their own keys for forward-compat + # (no rows exist yet for them; fall back to default). +} + + +def _open_session() -> object: + """Return a new SessionLocal context-manager instance. + + Deferred import avoids triggering Settings() validation at module import + time (unit tests run without DATABASE_URL env var). + Extracted as a named function so tests can patch it. + """ + from app.core.db import SessionLocal + + return SessionLocal() + + +def get_scraper_delay(source: str) -> float: + """Return the configured request_delay_sec for `source`. + + - Maps Yandex sub-scrapers to the umbrella 'yandex' key. + - Cached 60s in-process to avoid per-instance DB query. + - Any DB error -> returns the hardcoded default for that source. + """ + key = _KEY_ALIASES.get(source, source) + + now = time.time() + with _CACHE_LOCK: + cached = _CACHE.get(key) + if cached is not None and now - cached[1] < _CACHE_TTL_SEC: + return cached[0] + + try: + with _open_session() as db: + row = db.execute( + text( + "SELECT request_delay_sec FROM scraper_settings WHERE source = :s" + ), + {"s": key}, + ).first() + if row is not None: + value = float(row[0]) + else: + value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY) + except Exception as e: + logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e) + value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY) + + with _CACHE_LOCK: + _CACHE[key] = (value, now) + return value + + +def invalidate_cache(source: str | None = None) -> None: + """Clear the cache entry for `source`, or the whole cache if None. + + Called by admin endpoint after PUT to force a re-read on next get_scraper_delay(). + """ + with _CACHE_LOCK: + if source is None: + _CACHE.clear() + else: + key = _KEY_ALIASES.get(source, source) + _CACHE.pop(key, None) diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_detail.py b/tradein-mvp/backend/app/services/scrapers/yandex_detail.py index 4a4985ec..0d4c6101 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_detail.py @@ -15,6 +15,7 @@ from typing import Any from pydantic import BaseModel, Field from selectolax.parser import HTMLParser, Node +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper from app.services.scrapers.yandex_helpers import ( RE_AGENCY_FOUNDED, @@ -103,7 +104,11 @@ class YandexDetailScraper(BaseScraper): name = "yandex_detail" base_url = "https://realty.yandex.ru" - request_delay_sec = 4.0 + request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings + + def __init__(self) -> None: + super().__init__() + self.request_delay_sec = get_scraper_delay(self.name) # BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError async def fetch_around( diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py b/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py index cdca4db5..9139e77a 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py @@ -14,6 +14,7 @@ from typing import Any from pydantic import BaseModel, Field from selectolax.parser import HTMLParser, Node +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper from app.services.scrapers.yandex_helpers import ( RE_JK_ID, @@ -122,7 +123,11 @@ _WORD_NUM: dict[str, int] = { class YandexNewbuildingScraper(BaseScraper): name = "yandex_realty_nb" base_url = "https://realty.yandex.ru" - request_delay_sec = 4.0 + request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings + + def __init__(self) -> None: + super().__init__() + self.request_delay_sec = get_scraper_delay(self.name) async def fetch_around( self, lat: float, lon: float, radius_m: int = 1000 diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py index 4f0cf654..cae46c71 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_realty.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_realty.py @@ -18,6 +18,7 @@ from typing import Any from selectolax.parser import HTMLParser, Node +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper, ScrapedLot from app.services.scrapers.yandex_helpers import ( RE_FLOOR, @@ -48,6 +49,8 @@ class YandexRealtyScraper(BaseScraper): def __init__(self, city: str = DEFAULT_CITY) -> None: super().__init__() self.city = city + # Load global Yandex delay from DB at instantiation + self.request_delay_sec = get_scraper_delay(self.name) async def fetch_around( self, diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py index daa3606c..425a01f0 100644 --- a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py +++ b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py @@ -25,6 +25,7 @@ from urllib.parse import urlencode from pydantic import BaseModel, Field from selectolax.parser import HTMLParser +from app.services.scraper_settings import get_scraper_delay from app.services.scrapers.base import BaseScraper from app.services.scrapers.yandex_helpers import ( parse_dmy, @@ -119,7 +120,11 @@ class YandexValuationScraper(BaseScraper): name = "yandex_valuation" base_url = "https://realty.yandex.ru" valuation_path = "/otsenka-kvartiry-po-adresu-onlayn/" - request_delay_sec = 4.0 + request_delay_sec = 5.0 # class default; instance value loaded from scraper_settings + + def __init__(self) -> None: + super().__init__() + self.request_delay_sec = get_scraper_delay(self.name) async def fetch_around( self, lat: float, lon: float, radius_m: int = 1000 diff --git a/tradein-mvp/backend/data/sql/053_scraper_settings.sql b/tradein-mvp/backend/data/sql/053_scraper_settings.sql new file mode 100644 index 00000000..c65db0cf --- /dev/null +++ b/tradein-mvp/backend/data/sql/053_scraper_settings.sql @@ -0,0 +1,28 @@ +-- 053_scraper_settings.sql +-- Purpose: Per-source live-config for scraper request delays (anti-ban knob). +-- Loaded by app at runtime; UI can update without restart. +-- Dependencies: none (standalone settings table) +-- Sources: User request 2026-05-23 -- Yandex admin page + global delay knob. + +BEGIN; + +CREATE TABLE IF NOT EXISTS scraper_settings ( + source text PRIMARY KEY, -- 'yandex' / 'avito' / 'cian' / 'n1' / 'yandex_detail' / ... + request_delay_sec numeric(5,2) NOT NULL, -- seconds between requests (anti-ban). Range 1-60. + updated_at timestamptz NOT NULL DEFAULT NOW(), + -- Hint about who owns this knob (UI displays, no behavior) + description text +); + +COMMENT ON TABLE scraper_settings IS + 'Per-source runtime config for scrapers. Currently exposes request_delay_sec ' + '(anti-ban delay between HTTP requests). Editable via admin UI; backend reads ' + 'with 60s in-process cache. Source key matches BaseScraper.name in code.'; + +-- Seed: Yandex umbrella key applies to all 4 Yandex scrapers +-- (yandex / yandex_detail / yandex_newbuilding / yandex_valuation read this key). +INSERT INTO scraper_settings (source, request_delay_sec, description) +VALUES ('yandex', 5.0, 'Global delay applied to all Yandex Realty scrapers (SERP, detail, newbuilding, valuation)') +ON CONFLICT (source) DO NOTHING; + +COMMIT; diff --git a/tradein-mvp/backend/tests/test_scraper_settings.py b/tradein-mvp/backend/tests/test_scraper_settings.py new file mode 100644 index 00000000..f0d4cf6c --- /dev/null +++ b/tradein-mvp/backend/tests/test_scraper_settings.py @@ -0,0 +1,118 @@ +"""Tests for scraper_settings loader (cache + DB fallback + alias mapping).""" +from unittest.mock import MagicMock, patch + +import pytest + +from app.services import scraper_settings as ss + +# Patch target: _open_session is a module-level wrapper so tests can mock DB +_PATCH_TARGET = "app.services.scraper_settings._open_session" + + +@pytest.fixture(autouse=True) +def _reset_cache(): + ss._CACHE.clear() + yield + ss._CACHE.clear() + + +def _mock_session(row_value: float | None) -> MagicMock: + """Return a mock that behaves like SessionLocal() context-manager.""" + session = MagicMock() + session.__enter__ = MagicMock(return_value=session) + session.__exit__ = MagicMock(return_value=False) + if row_value is None: + session.execute.return_value.first.return_value = None + else: + session.execute.return_value.first.return_value = (row_value,) + # _open_session() returns the session directly (no extra call) + mock_open = MagicMock(return_value=session) + return mock_open, session + + +def test_returns_db_value_for_yandex(): + mock_open, _ = _mock_session(7.5) + with patch(_PATCH_TARGET, mock_open): + assert ss.get_scraper_delay("yandex") == 7.5 + + +def test_yandex_subkeys_map_to_yandex_umbrella(): + """yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row.""" + mock_open, _session = _mock_session(8.0) + with patch(_PATCH_TARGET, mock_open): + assert ss.get_scraper_delay("yandex_detail") == 8.0 + assert ss.get_scraper_delay("yandex_newbuilding") == 8.0 + assert ss.get_scraper_delay("yandex_valuation") == 8.0 + # All 3 lookups hit the cache after the first; _open_session called once + assert mock_open.call_count == 1 + + +def test_cache_hit_within_ttl(): + mock_open, _session = _mock_session(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex") + ss.get_scraper_delay("yandex") + ss.get_scraper_delay("yandex") + assert mock_open.call_count == 1 # cached after first + + +def test_cache_expires_after_ttl(): + mock_open, _session = _mock_session(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex") + # Force-age the cache + with ss._CACHE_LOCK: + v, t = ss._CACHE["yandex"] + ss._CACHE["yandex"] = (v, t - 999) + ss.get_scraper_delay("yandex") + assert mock_open.call_count == 2 + + +def test_db_missing_row_returns_default(): + mock_open, _ = _mock_session(None) + with patch(_PATCH_TARGET, mock_open): + assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"] + + +def test_db_error_returns_default(): + bad_session = MagicMock() + bad_session.__enter__ = MagicMock(return_value=bad_session) + bad_session.__exit__ = MagicMock(return_value=False) + bad_session.execute.side_effect = RuntimeError("db down") + bad_open = MagicMock(return_value=bad_session) + with patch(_PATCH_TARGET, bad_open): + assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"] + + +def test_unknown_source_uses_global_default(): + mock_open, _ = _mock_session(None) + with patch(_PATCH_TARGET, mock_open): + assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY + + +def test_invalidate_cache_specific_source(): + mock_open, _ = _mock_session(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex") + ss.invalidate_cache("yandex") + ss.get_scraper_delay("yandex") + assert mock_open.call_count == 2 + + +def test_invalidate_cache_all(): + mock_open, _ = _mock_session(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex") + ss.invalidate_cache() + ss.get_scraper_delay("yandex") + assert mock_open.call_count == 2 + + +def test_invalidate_alias_invalidates_umbrella(): + """invalidate_cache('yandex_detail') should clear the 'yandex' umbrella key.""" + mock_open, _ = _mock_session(6.0) + with patch(_PATCH_TARGET, mock_open): + ss.get_scraper_delay("yandex_detail") + ss.invalidate_cache("yandex_detail") + ss.get_scraper_delay("yandex_detail") + assert mock_open.call_count == 2 diff --git a/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py b/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py new file mode 100644 index 00000000..3b262ebc --- /dev/null +++ b/tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py @@ -0,0 +1,35 @@ +"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings.""" +from unittest.mock import patch + +from app.services.scrapers.yandex_detail import YandexDetailScraper +from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper +from app.services.scrapers.yandex_realty import YandexRealtyScraper +from app.services.scrapers.yandex_valuation import YandexValuationScraper + + +def test_yandex_realty_picks_up_delay(): + with patch("app.services.scrapers.yandex_realty.get_scraper_delay", return_value=9.5): + s = YandexRealtyScraper() + assert s.request_delay_sec == 9.5 + + +def test_yandex_detail_picks_up_delay(): + with patch("app.services.scrapers.yandex_detail.get_scraper_delay", return_value=9.5): + s = YandexDetailScraper() + assert s.request_delay_sec == 9.5 + + +def test_yandex_newbuilding_picks_up_delay(): + with patch( + "app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5 + ): + s = YandexNewbuildingScraper() + assert s.request_delay_sec == 9.5 + + +def test_yandex_valuation_picks_up_delay(): + with patch( + "app.services.scrapers.yandex_valuation.get_scraper_delay", return_value=9.5 + ): + s = YandexValuationScraper() + assert s.request_delay_sec == 9.5