Merge remote-tracking branch 'forgejo/main' into feat/tradein-global-scraper-delay

# Conflicts:
#	tradein-mvp/backend/app/services/scraper_settings.py
#	tradein-mvp/backend/data/sql/053_scraper_settings.sql
#	tradein-mvp/backend/tests/test_scraper_settings.py

tradein-mvp/backend/app/api/v1/admin.py

@@ -35,7 +35,10 @@ from app.services.scrapers.avito_imv import (
 from app.services.scrapers.base import save_listings
 from app.services.scrapers.cian import CianScraper
 from app.services.scrapers.n1 import N1Scraper
+from app.services.scrapers.yandex_detail import YandexDetailScraper
+from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
 from app.services.scrapers.yandex_realty import YandexRealtyScraper
+from app.services.scrapers.yandex_valuation import YandexValuationScraper
 
 logger = logging.getLogger(__name__)
 
@@ -701,3 +704,122 @@ def update_schedule(
         next_run_at=row["next_run_at"].isoformat() if row["next_run_at"] else None,
         updated_at=row["updated_at"].isoformat() if row["updated_at"] else None,
     )
+
+
+# -- Yandex ad-hoc scrape triggers --------------------------------------------
+
+
+class YandexDetailTriggerResp(BaseModel):
+    ok: bool
+    offer_url: str
+    offer_id: str | None
+    price_rub: int | None
+    title: str | None
+    photo_count: int
+
+
+@router.post("/scrape/yandex-detail", response_model=YandexDetailTriggerResp)
+async def scrape_yandex_detail(
+    offer_url: str,
+) -> YandexDetailTriggerResp:
+    """Ad-hoc parse one Yandex offer detail page.
+
+    Returns a snapshot of the extracted DetailEnrichment (no DB write - read-only
+    debug; main pipeline writes via estimator on /estimate flow).
+    """
+    async with YandexDetailScraper() as scraper:
+        result = await scraper.fetch_detail(offer_url)
+    if result is None:
+        raise HTTPException(404, f"Could not parse Yandex detail page: {offer_url}")
+    return YandexDetailTriggerResp(
+        ok=True,
+        offer_url=offer_url,
+        offer_id=result.offer_id,
+        price_rub=result.price_rub,
+        title=result.title,
+        photo_count=len(result.photo_urls),
+    )
+
+
+class YandexNewbuildingTriggerResp(BaseModel):
+    ok: bool
+    ext_id: str
+    ext_slug: str
+    name: str | None
+    lat: float | None
+    lon: float | None
+    rating: float | None
+    ratings_count: int | None
+    text_reviews_count: int | None
+    developer_name: str | None
+
+
+@router.post("/scrape/yandex-newbuilding", response_model=YandexNewbuildingTriggerResp)
+async def scrape_yandex_newbuilding(
+    slug: str,
+    id: str,
+    city: str = "ekaterinburg",
+) -> YandexNewbuildingTriggerResp:
+    """Ad-hoc parse a Yandex JK landing page.
+
+    URL: /{city}/kupit/novostrojka/<slug>-<id>/
+    """
+    async with YandexNewbuildingScraper() as scraper:
+        result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city)
+    if result is None:
+        raise HTTPException(404, f"Could not parse Yandex JK: {slug}-{id} in {city}")
+    return YandexNewbuildingTriggerResp(
+        ok=True,
+        ext_id=result.ext_id,
+        ext_slug=result.ext_slug,
+        name=result.name,
+        lat=result.lat,
+        lon=result.lon,
+        rating=result.rating,
+        ratings_count=result.ratings_count,
+        text_reviews_count=result.text_reviews_count,
+        developer_name=result.developer_name,
+    )
+
+
+class YandexValuationTriggerResp(BaseModel):
+    ok: bool
+    address: str
+    year_built: int | None
+    total_floors: int | None
+    house_type: str | None
+    has_lift: bool | None
+    total_objects: int | None
+    history_items_count: int
+
+
+@router.post("/scrape/yandex-valuation", response_model=YandexValuationTriggerResp)
+async def scrape_yandex_valuation(
+    address: str,
+    offer_category: str = "APARTMENT",
+    offer_type: str = "SELL",
+    page: int = 1,
+) -> YandexValuationTriggerResp:
+    """Ad-hoc fetch Yandex Valuation house-history for an address (debug, no cache write).
+
+    Anonymous GET - works without auth.
+    """
+    async with YandexValuationScraper() as scraper:
+        result = await scraper.fetch_house_history(
+            address=address,
+            offer_category=offer_category,
+            offer_type=offer_type,
+            page=page,
+        )
+    if result is None:
+        raise HTTPException(404, f"Could not parse Yandex Valuation: {address}")
+    return YandexValuationTriggerResp(
+        ok=True,
+        address=address,
+        year_built=result.house.year_built,
+        total_floors=result.house.total_floors,
+        house_type=result.house.house_type,
+        has_lift=result.house.has_lift,
+        total_objects=result.house.total_objects,
+        history_items_count=len(result.history_items),
+    )

tradein-mvp/backend/app/services/scraper_settings.py

@@ -12,60 +12,66 @@ invalidate_cache() вызывается из admin API PUT для немедле
 from __future__ import annotations
 
 import logging
-import threading
 import time
-from collections.abc import Generator
+from threading import Lock
-from contextlib import contextmanager
 
 from sqlalchemy import text
-from sqlalchemy.orm import Session
-
-from app.core.db import SessionLocal
 
 logger = logging.getLogger(__name__)
 
 # Специальный ключ глобальной задержки (строка в scraper_settings с этим source).
 _GLOBAL_KEY = "global"
 
-# TTL кеша в секундах — после истечения перечитывается из БД.
+# In-process cache: {source: (delay_sec, fetched_at_epoch)}
+_CACHE: dict[str, tuple[float, float]] = {}
+_CACHE_LOCK = Lock()
 _CACHE_TTL_SEC = 60.0
 
-# Дефолтные задержки если строка в БД отсутствует.
+# Default fallback if DB row missing or DB unreachable
 _DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
     "avito": 7.0,
     "cian": 5.0,
     "n1": 5.0,
     "yandex": 5.0,
+    "yandex_detail": 5.0,
+    "yandex_newbuilding": 5.0,
+    "yandex_valuation": 5.0,
     "domrf": 5.0,
     "rosreestr": 5.0,
 }
-
-# Fallback для неизвестного источника.
 _GLOBAL_DEFAULT_DELAY = 5.0
 
-# Алиасы источников: ключ → canonical source (можно расширять без изменения вызывающего кода).
+# Yandex sub-scrapers all read the umbrella "yandex" key.
-_KEY_ALIASES: dict[str, str] = {}
+# Map: scraper_name -> DB row key.
-
+_KEY_ALIASES: dict[str, str] = {
-# Кеш: source → (delay_value, timestamp).
+    "yandex": "yandex",
-_CACHE: dict[str, tuple[float, float]] = {}
+    "yandex_detail": "yandex",
-_CACHE_LOCK = threading.Lock()
+    "yandex_newbuilding": "yandex",
+    "yandex_realty_nb": "yandex",  # BaseScraper.name used by YandexNewbuildingScraper
+    "yandex_valuation": "yandex",
+    # Non-Yandex sources resolve to their own keys for forward-compat
+    # (no rows exist yet for them; fall back to default).
+}
 
 
-@contextmanager
+def _open_session() -> object:
-def _open_session() -> Generator[Session, None, None]:
+    """Return a new SessionLocal context-manager instance.
-    """Открыть сессию БД с гарантированным закрытием."""
+
-    db = SessionLocal()
+    Deferred import avoids triggering Settings() validation at module import
-    try:
+    time (unit tests run without DATABASE_URL env var).
-        yield db
+    Extracted as a named function so tests can patch it.
-    finally:
+    """
-        db.close()
+    from app.core.db import SessionLocal
+
+    return SessionLocal()
 
 
 def _get_setting_cached(key: str) -> float:
-    """Кеш-aware чтение одного source из scraper_settings.
+    """Cache-aware read of one source key from scraper_settings.
 
-    Возвращает request_delay_sec из БД (с кешем TTL=60s).
+    Returns request_delay_sec from DB (with TTL=60s cache).
-    При отсутствии строки — фолбек на _DEFAULT_DELAY_BY_SOURCE или 0.0 для global.
+    Missing row: falls back to _DEFAULT_DELAY_BY_SOURCE, or 0.0 for _GLOBAL_KEY
+    (meaning no global floor is applied when the row hasn't been seeded yet).
     """
     now = time.time()
     with _CACHE_LOCK:
@@ -76,20 +82,20 @@ def _get_setting_cached(key: str) -> float:
     try:
         with _open_session() as db:
             row = db.execute(
-                text("SELECT request_delay_sec FROM scraper_settings WHERE source = :s"),
+                text(
+                    "SELECT request_delay_sec FROM scraper_settings WHERE source = :s"
+                ),
                 {"s": key},
             ).first()
         if row is not None:
             value = float(row[0])
         elif key == _GLOBAL_KEY:
-            # Глобальная строка ещё не создана — не применяем floor.
+            # Global row not yet seeded — don't apply floor.
             value = 0.0
         else:
             value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
-    except Exception as exc:
+    except Exception as e:
-        logger.warning(
+        logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
-            "scraper_settings: load failed for %s — using default: %s", key, exc
-        )
         value = 0.0 if key == _GLOBAL_KEY else _DEFAULT_DELAY_BY_SOURCE.get(
             key, _GLOBAL_DEFAULT_DELAY
         )
@@ -100,10 +106,13 @@ def _get_setting_cached(key: str) -> float:
 
 
 def get_scraper_delay(source: str) -> float:
-    """Вернуть эффективную задержку для source = max(per_source, global).
+    """Return the configured request_delay_sec for `source`.
 
-    Строка source='global' задаёт нижнюю планку для всех парсеров.
+    Effective delay = max(per_source, global).
-    Если global=0 — используется только per-source значение.
+    - Maps Yandex sub-scrapers to the umbrella 'yandex' key.
+    - Cached 60s in-process to avoid per-instance DB query.
+    - source='global' sets a floor across all scrapers (0 = disabled).
+    - Any DB error -> returns the hardcoded default for that source.
     """
     key = _KEY_ALIASES.get(source, source)
     per_source = _get_setting_cached(key)
@@ -112,19 +121,18 @@ def get_scraper_delay(source: str) -> float:
 
 
 def invalidate_cache(source: str | None = None) -> None:
-    """Сбросить кеш для source (или весь кеш если source=None).
+    """Clear the cache entry for `source`, or the whole cache if None.
 
-    Вызывается из admin API PUT /scraper-settings/{source} для немедленного
+    Called by admin endpoint after PUT to force a re-read on next get_scraper_delay().
-    применения нового значения без ожидания TTL.
     """
     with _CACHE_LOCK:
         if source is None:
             _CACHE.clear()
             logger.info("scraper_settings: cache cleared (all sources)")
         else:
-            _CACHE.pop(source, None)
+            key = _KEY_ALIASES.get(source, source)
-            # Если source — алиас, сбрасываем canonical key тоже.
+            _CACHE.pop(key, None)
-            canonical = _KEY_ALIASES.get(source)
+            # Also clear the source itself if it differed from key (alias case).
-            if canonical:
+            if key != source:
-                _CACHE.pop(canonical, None)
+                _CACHE.pop(source, None)
             logger.info("scraper_settings: cache cleared for source=%s", source)

tradein-mvp/backend/app/services/scrapers/yandex_detail.py

@@ -15,6 +15,7 @@ from typing import Any
 from pydantic import BaseModel, Field
 from selectolax.parser import HTMLParser, Node
 
+from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper
 from app.services.scrapers.yandex_helpers import (
     RE_AGENCY_FOUNDED,
@@ -103,7 +104,11 @@ class YandexDetailScraper(BaseScraper):
 
     name = "yandex_detail"
     base_url = "https://realty.yandex.ru"
-    request_delay_sec = 4.0
+    request_delay_sec = 5.0  # class default; instance value loaded from scraper_settings
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.request_delay_sec = get_scraper_delay(self.name)
 
     # BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError
     async def fetch_around(

tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py

@@ -14,6 +14,7 @@ from typing import Any
 from pydantic import BaseModel, Field
 from selectolax.parser import HTMLParser, Node
 
+from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper
 from app.services.scrapers.yandex_helpers import (
     RE_JK_ID,
@@ -122,7 +123,11 @@ _WORD_NUM: dict[str, int] = {
 class YandexNewbuildingScraper(BaseScraper):
     name = "yandex_realty_nb"
     base_url = "https://realty.yandex.ru"
-    request_delay_sec = 4.0
+    request_delay_sec = 5.0  # class default; instance value loaded from scraper_settings
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.request_delay_sec = get_scraper_delay(self.name)
 
     async def fetch_around(
         self, lat: float, lon: float, radius_m: int = 1000

tradein-mvp/backend/app/services/scrapers/yandex_realty.py

@@ -18,6 +18,7 @@ from typing import Any
 
 from selectolax.parser import HTMLParser, Node
 
+from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper, ScrapedLot
 from app.services.scrapers.yandex_helpers import (
     RE_FLOOR,
@@ -48,6 +49,8 @@ class YandexRealtyScraper(BaseScraper):
     def __init__(self, city: str = DEFAULT_CITY) -> None:
         super().__init__()
         self.city = city
+        # Load global Yandex delay from DB at instantiation
+        self.request_delay_sec = get_scraper_delay(self.name)
 
     async def fetch_around(
         self,

tradein-mvp/backend/app/services/scrapers/yandex_valuation.py

@@ -25,6 +25,7 @@ from urllib.parse import urlencode
 from pydantic import BaseModel, Field
 from selectolax.parser import HTMLParser
 
+from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper
 from app.services.scrapers.yandex_helpers import (
     parse_dmy,
@@ -119,7 +120,11 @@ class YandexValuationScraper(BaseScraper):
     name = "yandex_valuation"
     base_url = "https://realty.yandex.ru"
     valuation_path = "/otsenka-kvartiry-po-adresu-onlayn/"
-    request_delay_sec = 4.0
+    request_delay_sec = 5.0  # class default; instance value loaded from scraper_settings
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.request_delay_sec = get_scraper_delay(self.name)
 
     async def fetch_around(
         self, lat: float, lon: float, radius_m: int = 1000

tradein-mvp/backend/data/sql/053_scraper_settings.sql

@@ -1,19 +1,28 @@
 -- 053_scraper_settings.sql
--- Таблица настроек парсеров: задержки между запросами (anti-ban), extensible.
+-- Purpose: Per-source live-config for scraper request delays (anti-ban knob).
--- Одна строка на источник (source); UNIQUE(source).
+--          Loaded by app at runtime; UI can update without restart.
+-- Dependencies: none (standalone settings table)
+-- Sources: User request 2026-05-23 -- Yandex admin page + global delay knob.
+
 BEGIN;
 
 CREATE TABLE IF NOT EXISTS scraper_settings (
-    id               SERIAL PRIMARY KEY,
+    source          text PRIMARY KEY,         -- 'yandex' / 'avito' / 'cian' / 'n1' / 'yandex_detail' / ...
-    source           TEXT        NOT NULL,
+    request_delay_sec numeric(5,2) NOT NULL,  -- seconds between requests (anti-ban). Range 1-60.
-    request_delay_sec NUMERIC(6,2) NOT NULL DEFAULT 5.0,
+    updated_at      timestamptz NOT NULL DEFAULT NOW(),
-    description      TEXT,
+    -- Hint about who owns this knob (UI displays, no behavior)
-    created_at       TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+    description     text
-    updated_at       TIMESTAMPTZ NOT NULL DEFAULT NOW(),
-    CONSTRAINT scraper_settings_source_uq UNIQUE (source)
 );
 
 COMMENT ON TABLE scraper_settings IS
-    'Per-source scraper delay settings. Special source=''global'' acts as floor across all scrapers.';
+  'Per-source runtime config for scrapers. Currently exposes request_delay_sec '
+  '(anti-ban delay between HTTP requests). Editable via admin UI; backend reads '
+  'with 60s in-process cache. Source key matches BaseScraper.name in code.';
+
+-- Seed: Yandex umbrella key applies to all 4 Yandex scrapers
+-- (yandex / yandex_detail / yandex_newbuilding / yandex_valuation read this key).
+INSERT INTO scraper_settings (source, request_delay_sec, description)
+VALUES ('yandex', 5.0, 'Global delay applied to all Yandex Realty scrapers (SERP, detail, newbuilding, valuation)')
+ON CONFLICT (source) DO NOTHING;
 
 COMMIT;

tradein-mvp/backend/tests/test_scraper_settings.py

@@ -1,6 +1,9 @@
-"""Offline tests for scraper_settings service (no DB required).
+"""Tests for scraper_settings loader (cache + DB fallback + alias mapping + global delay).
 
-Тестируем логику max(per_source, global) через монкей-патч _get_setting_cached.
+Covers:
+  - Per-source value lookup + caching (main's tests)
+  - Global delay max() logic (our additions)
+  - Admin endpoint smoke
 """
 
 import os
@@ -12,15 +15,49 @@ from unittest.mock import MagicMock, patch
 import pytest
 
 import app.services.scraper_settings as ss_mod
+from app.services import scraper_settings as ss
 from app.services.scraper_settings import get_scraper_delay, invalidate_cache
 
+# Patch target: _open_session is a module-level wrapper so tests can mock DB
+_PATCH_TARGET = "app.services.scraper_settings._open_session"
 
-# ── helpers ──────────────────────────────────────────────────────────────────
+
+# ── Fixtures ──────────────────────────────────────────────────────────────────
+
+
+@pytest.fixture(autouse=True)
+def _reset_cache():
+    ss._CACHE.clear()
+    yield
+    ss._CACHE.clear()
+
+
+# ── Helpers ───────────────────────────────────────────────────────────────────
+
+
+def _mock_session_uniform(row_value: float | None) -> tuple[MagicMock, MagicMock]:
+    """Mock that returns the same value for any key (context-manager style).
+
+    Used by tests that don't care about multi-key distinction.
+    Global key will also return the same value, so max() returns row_value.
+    """
+    session = MagicMock()
+    session.__enter__ = MagicMock(return_value=session)
+    session.__exit__ = MagicMock(return_value=False)
+    if row_value is None:
+        session.execute.return_value.first.return_value = None
+    else:
+        session.execute.return_value.first.return_value = (row_value,)
+    mock_open = MagicMock(return_value=session)
+    return mock_open, session
 
 
 def _make_mock_session(values: dict[str, float | None]) -> MagicMock:
-    """Создать mock сессии БД с фиксированными значениями per source key."""
+    """Mock сессии БД с фиксированными значениями per source key.
 
+    Поддерживает разные значения для разных ключей — используется в тестах
+    global delay логики.
+    """
     mock_db = MagicMock()
 
     def fake_execute(sql, params):
@@ -36,11 +73,112 @@ def _make_mock_session(values: dict[str, float | None]) -> MagicMock:
     return mock_db
 
 
-# ── tests ─────────────────────────────────────────────────────────────────────
+# ── Main's tests: core cache + alias + fallback behaviour ─────────────────────
+
+
+def test_returns_db_value_for_yandex():
+    """DB value 7.5, global also 7.5 → max(7.5, 7.5) = 7.5."""
+    mock_open, _ = _mock_session_uniform(7.5)
+    with patch(_PATCH_TARGET, mock_open):
+        assert ss.get_scraper_delay("yandex") == 7.5
+
+
+def test_yandex_subkeys_map_to_yandex_umbrella():
+    """yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row."""
+    mock_open, _session = _mock_session_uniform(8.0)
+    with patch(_PATCH_TARGET, mock_open):
+        assert ss.get_scraper_delay("yandex_detail") == 8.0
+        assert ss.get_scraper_delay("yandex_newbuilding") == 8.0
+        assert ss.get_scraper_delay("yandex_valuation") == 8.0
+    # First call fetches yandex + global (2 DB calls). Remaining 2 subkey calls
+    # map to "yandex" (already cached) and "global" (already cached) → 0 more.
+    assert mock_open.call_count == 2
+
+
+def test_cache_hit_within_ttl():
+    """After first call both yandex+global are cached; repeats hit cache only."""
+    mock_open, _session = _mock_session_uniform(6.0)
+    with patch(_PATCH_TARGET, mock_open):
+        ss.get_scraper_delay("yandex")
+        ss.get_scraper_delay("yandex")
+        ss.get_scraper_delay("yandex")
+    # First call: 2 DB hits (yandex + global). Subsequent: 0.
+    assert mock_open.call_count == 2
+
+
+def test_cache_expires_after_ttl():
+    """Force-age yandex key — next call re-fetches yandex only (global still fresh)."""
+    mock_open, _session = _mock_session_uniform(6.0)
+    with patch(_PATCH_TARGET, mock_open):
+        ss.get_scraper_delay("yandex")  # +2 calls (yandex + global)
+        # Force-age the yandex entry
+        with ss._CACHE_LOCK:
+            v, t = ss._CACHE["yandex"]
+            ss._CACHE["yandex"] = (v, t - 999)
+        ss.get_scraper_delay("yandex")  # +1 call (yandex stale; global still fresh)
+    assert mock_open.call_count == 3
+
+
+def test_db_missing_row_returns_default():
+    """Missing yandex row → _DEFAULT_DELAY_BY_SOURCE. Global None → 0.0. max = default."""
+    mock_open, _ = _mock_session_uniform(None)
+    with patch(_PATCH_TARGET, mock_open):
+        assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]
+
+
+def test_db_error_returns_default():
+    """DB error → hardcoded default. Global error → 0.0. max = default."""
+    bad_session = MagicMock()
+    bad_session.__enter__ = MagicMock(return_value=bad_session)
+    bad_session.__exit__ = MagicMock(return_value=False)
+    bad_session.execute.side_effect = RuntimeError("db down")
+    bad_open = MagicMock(return_value=bad_session)
+    with patch(_PATCH_TARGET, bad_open):
+        assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]
+
+
+def test_unknown_source_uses_global_default():
+    """Unknown source, DB row None → _GLOBAL_DEFAULT_DELAY. Global None → 0.0. max = 5.0."""
+    mock_open, _ = _mock_session_uniform(None)
+    with patch(_PATCH_TARGET, mock_open):
+        assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY
+
+
+def test_invalidate_cache_specific_source():
+    """invalidate_cache(yandex) drops yandex entry; next call re-fetches yandex only."""
+    mock_open, _ = _mock_session_uniform(6.0)
+    with patch(_PATCH_TARGET, mock_open):
+        ss.get_scraper_delay("yandex")  # +2 (yandex + global)
+        ss.invalidate_cache("yandex")   # clears yandex key
+        ss.get_scraper_delay("yandex")  # +1 (yandex stale; global still cached)
+    assert mock_open.call_count == 3
+
+
+def test_invalidate_cache_all():
+    """invalidate_cache() clears all entries; next call re-fetches both."""
+    mock_open, _ = _mock_session_uniform(6.0)
+    with patch(_PATCH_TARGET, mock_open):
+        ss.get_scraper_delay("yandex")  # +2 (yandex + global)
+        ss.invalidate_cache()           # clears everything
+        ss.get_scraper_delay("yandex")  # +2 (both stale)
+    assert mock_open.call_count == 4
+
+
+def test_invalidate_alias_invalidates_umbrella():
+    """invalidate_cache('yandex_detail') clears the 'yandex' umbrella key."""
+    mock_open, _ = _mock_session_uniform(6.0)
+    with patch(_PATCH_TARGET, mock_open):
+        ss.get_scraper_delay("yandex_detail")  # +2 (yandex + global)
+        ss.invalidate_cache("yandex_detail")    # clears yandex (via alias)
+        ss.get_scraper_delay("yandex_detail")   # +1 (yandex stale; global fresh)
+    assert mock_open.call_count == 3
+
+
+# ── Our additions: global delay max() logic ───────────────────────────────────
 
 
 def test_global_delay_takes_precedence_when_higher(monkeypatch):
-    """global=10 > avito=7 → возвращает 10."""
+    """global=10 > avito=7 → returns 10."""
     invalidate_cache()
 
     mock_db = _make_mock_session({"avito": 7.0, "global": 10.0})
@@ -51,7 +189,7 @@ def test_global_delay_takes_precedence_when_higher(monkeypatch):
 
 
 def test_per_source_takes_precedence_when_higher(monkeypatch):
-    """avito=7 > global=2 → возвращает 7."""
+    """avito=7 > global=2 → returns 7."""
     invalidate_cache()
 
     mock_db = _make_mock_session({"avito": 7.0, "global": 2.0})
@@ -62,7 +200,7 @@ def test_per_source_takes_precedence_when_higher(monkeypatch):
 
 
 def test_global_zero_means_no_floor(monkeypatch):
-    """global=0 → используется только per-source значение."""
+    """global=0 → only per-source value applies."""
     invalidate_cache()
 
     mock_db = _make_mock_session({"cian": 5.0, "global": 0.0})
@@ -73,80 +211,18 @@ def test_global_zero_means_no_floor(monkeypatch):
 
 
 def test_missing_per_source_row_uses_class_default(monkeypatch):
-    """Если строки source в БД нет — фолбек на _DEFAULT_DELAY_BY_SOURCE."""
+    """Missing source row → _DEFAULT_DELAY_BY_SOURCE. global=0 → no floor."""
     invalidate_cache()
 
-    # n1 строки нет в БД, global=0
     mock_db = _make_mock_session({"n1": None, "global": 0.0})
     monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
 
     delay = get_scraper_delay("n1")
-    # Дефолт для n1 = 5.0
     assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["n1"]
 
 
-def test_unknown_source_uses_global_default(monkeypatch):
-    """Неизвестный source без строки в БД — _GLOBAL_DEFAULT_DELAY."""
-    invalidate_cache()
-
-    mock_db = _make_mock_session({"some_new_source": None, "global": 0.0})
-    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
-
-    delay = get_scraper_delay("some_new_source")
-    assert delay == ss_mod._GLOBAL_DEFAULT_DELAY
-
-
-def test_db_error_falls_back_to_default(monkeypatch):
-    """Ошибка БД → graceful fallback на дефолт, не crash."""
-    invalidate_cache()
-
-    def broken_open():
-        raise RuntimeError("DB is down")
-
-    monkeypatch.setattr(ss_mod, "_open_session", broken_open)
-
-    # Не должно кидать исключение
-    delay = get_scraper_delay("avito")
-    # При ошибке: per_source → _DEFAULT_DELAY_BY_SOURCE['avito'] = 7.0, global → 0.0
-    assert delay == 7.0
-
-
-def test_invalidate_cache_single_source(monkeypatch):
-    """invalidate_cache(source) удаляет только указанный source."""
-    invalidate_cache()
-
-    mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0})
-    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
-
-    # Прогреть кеш для cian и avito
-    get_scraper_delay("cian")
-    get_scraper_delay("avito")
-
-    # Сбросить только cian
-    invalidate_cache("cian")
-
-    with ss_mod._CACHE_LOCK:
-        assert "cian" not in ss_mod._CACHE
-        assert "avito" in ss_mod._CACHE
-
-
-def test_invalidate_cache_all(monkeypatch):
-    """invalidate_cache() без аргумента очищает весь кеш."""
-    invalidate_cache()
-
-    mock_db = _make_mock_session({"avito": 7.0, "global": 0.0})
-    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
-
-    get_scraper_delay("avito")
-
-    invalidate_cache()
-
-    with ss_mod._CACHE_LOCK:
-        assert len(ss_mod._CACHE) == 0
-
-
 def test_global_missing_row_treated_as_zero(monkeypatch):
-    """Если строки 'global' нет в БД — фолбек 0.0 (не применяем floor)."""
+    """Missing 'global' row → fallback 0.0, avito=7.0 → max(7.0, 0.0) = 7.0."""
     invalidate_cache()
 
     mock_db = _make_mock_session({"avito": 7.0, "global": None})
@@ -156,23 +232,66 @@ def test_global_missing_row_treated_as_zero(monkeypatch):
     assert delay == 7.0
 
 
-# ── Admin API integration smoke ───────────────────────────────────────────────
+def test_db_error_falls_back_to_default_global_logic(monkeypatch):
+    """DB error → graceful fallback, no crash. avito default=7.0, global error→0.0."""
+    invalidate_cache()
+
+    def broken_open():
+        raise RuntimeError("DB is down")
+
+    monkeypatch.setattr(ss_mod, "_open_session", broken_open)
+
+    delay = get_scraper_delay("avito")
+    assert delay == ss_mod._DEFAULT_DELAY_BY_SOURCE["avito"]
+
+
+def test_invalidate_cache_single_source_global_logic(monkeypatch):
+    """invalidate_cache(source) removes only that source from cache."""
+    invalidate_cache()
+
+    mock_db = _make_mock_session({"cian": 5.0, "avito": 7.0, "global": 0.0})
+    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
+
+    get_scraper_delay("cian")
+    get_scraper_delay("avito")
+
+    invalidate_cache("cian")
+
+    with ss_mod._CACHE_LOCK:
+        assert "cian" not in ss_mod._CACHE
+        assert "avito" in ss_mod._CACHE
+
+
+def test_invalidate_cache_all_global_logic(monkeypatch):
+    """invalidate_cache() without args clears the entire cache."""
+    invalidate_cache()
+
+    mock_db = _make_mock_session({"avito": 7.0, "global": 0.0})
+    monkeypatch.setattr(ss_mod, "_open_session", lambda: mock_db)
+
+    get_scraper_delay("avito")
+    invalidate_cache()
+
+    with ss_mod._CACHE_LOCK:
+        assert len(ss_mod._CACHE) == 0
+
+
+# ── Admin API endpoint smoke ───────────────────────────────────────────────────
 
 
 def test_list_scraper_settings_endpoint(monkeypatch):
-    """GET /scraper-settings — эндпоинт монтируется, возвращает список."""
+    """GET /scraper-settings mounts and returns a list including 'global'."""
     import sys
-    from unittest.mock import MagicMock as MM
 
-    # Stub weasyprint (нет на CI)
+    # Stub weasyprint (not on CI)
-    _wp = MM()
+    _wp = MagicMock()
     sys.modules.setdefault("weasyprint", _wp)
 
+    from fastapi import FastAPI
     from fastapi.testclient import TestClient
 
     from app.api.v1 import admin as admin_module
     from app.core.db import get_db
-    from fastapi import FastAPI
 
     app = FastAPI()
     app.include_router(admin_module.router, prefix="/api/v1/admin")
@@ -182,7 +301,7 @@ def test_list_scraper_settings_endpoint(monkeypatch):
         {"source": "global", "request_delay_sec": 0.0, "description": "Global", "updated_at": None},
     ]
 
-    mock_db = MM()
+    mock_db = MagicMock()
     mock_db.execute.return_value.mappings.return_value.all.return_value = fake_rows
     app.dependency_overrides[get_db] = lambda: mock_db
 

tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py

@@ -0,0 +1,35 @@
+"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings."""
+from unittest.mock import patch
+
+from app.services.scrapers.yandex_detail import YandexDetailScraper
+from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
+from app.services.scrapers.yandex_realty import YandexRealtyScraper
+from app.services.scrapers.yandex_valuation import YandexValuationScraper
+
+
+def test_yandex_realty_picks_up_delay():
+    with patch("app.services.scrapers.yandex_realty.get_scraper_delay", return_value=9.5):
+        s = YandexRealtyScraper()
+        assert s.request_delay_sec == 9.5
+
+
+def test_yandex_detail_picks_up_delay():
+    with patch("app.services.scrapers.yandex_detail.get_scraper_delay", return_value=9.5):
+        s = YandexDetailScraper()
+        assert s.request_delay_sec == 9.5
+
+
+def test_yandex_newbuilding_picks_up_delay():
+    with patch(
+        "app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5
+    ):
+        s = YandexNewbuildingScraper()
+        assert s.request_delay_sec == 9.5
+
+
+def test_yandex_valuation_picks_up_delay():
+    with patch(
+        "app.services.scrapers.yandex_valuation.get_scraper_delay", return_value=9.5
+    ):
+        s = YandexValuationScraper()
+        assert s.request_delay_sec == 9.5