feat(tradein): scraper_settings -- live-config request delay for Yandex scrapers + admin triggers

Backend foundation for the Yandex admin scraper page (UI follows in next PR).

scraper_settings (mig 053):
- New table source PK + request_delay_sec numeric(5,2) + updated_at + description
- Seeded with row 'yandex' = 5.0s (umbrella key for all 4 Yandex scrapers)

scraper_settings.py (new):
- get_scraper_delay(source) -> float, with 60s in-process cache + DB fallback
- Yandex sub-scrapers (yandex_detail / yandex_newbuilding / yandex_valuation)
  resolve to umbrella 'yandex' key via _KEY_ALIASES
- invalidate_cache(source | None) for admin PUT side-effect
- Deferred SessionLocal import (_open_session helper) for unit-test compat

4 Yandex scrapers wired to load delay at __init__:
- yandex_realty.py / yandex_detail.py / yandex_newbuilding.py / yandex_valuation.py
- Class default bumped to 5.0; instance value overridden via get_scraper_delay(self.name)

admin.py -- 5 new endpoints:
- GET    /scraper-settings           list[ScraperSetting]
- PUT    /scraper-settings/{source}  upsert + invalidate_cache
- POST   /scrape/yandex-detail       offer_url= -> detail snapshot
- POST   /scrape/yandex-newbuilding  slug=, id=, city=ekaterinburg -> JK snapshot
- POST   /scrape/yandex-valuation    address=, offer_category=, offer_type=, page= -> house meta

Tests: 10 + 4 = 14 new unit tests (loader cache/alias/error paths + scraper wiring).
Ruff clean.

tradein-mvp/backend/app/api/v1/admin.py

@@ -22,6 +22,7 @@ from app.services import cian_session as cian_session_svc
 from app.services import scrape_runs as runs_mod
 from app.services.geocoder import geocode
 from app.services.scrape_pipeline import run_avito_city_sweep
+from app.services.scraper_settings import invalidate_cache
 from app.services.scrapers.avito import AvitoScraper
 from app.services.scrapers.avito_detail import fetch_detail, save_detail_enrichment
 from app.services.scrapers.avito_houses import fetch_house_catalog, save_house_catalog_enrichment
@@ -34,7 +35,10 @@ from app.services.scrapers.avito_imv import (
 from app.services.scrapers.base import save_listings
 from app.services.scrapers.cian import CianScraper
 from app.services.scrapers.n1 import N1Scraper
+from app.services.scrapers.yandex_detail import YandexDetailScraper
+from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
 from app.services.scrapers.yandex_realty import YandexRealtyScraper
+from app.services.scrapers.yandex_valuation import YandexValuationScraper
 
 logger = logging.getLogger(__name__)
 
@@ -75,6 +79,17 @@ class ScrapeResponse(BaseModel):
     by_source: list[ScrapeResult]
 
 
+class ScraperSetting(BaseModel):
+    source: str
+    request_delay_sec: float
+    description: str | None = None
+    updated_at: str | None = None  # ISO
+
+
+class ScraperSettingUpdate(BaseModel):
+    request_delay_sec: float = Field(ge=1.0, le=60.0)
+
+
 @router.post("/scrape", response_model=ScrapeResponse)
 async def scrape_around(
     payload: ScrapeRequest,
@@ -632,3 +647,194 @@ def update_schedule(
         next_run_at=row["next_run_at"].isoformat() if row["next_run_at"] else None,
         updated_at=row["updated_at"].isoformat() if row["updated_at"] else None,
     )
+
+
+# -- Scraper settings (per-source live config) --------------------------------
+
+
+@router.get("/scraper-settings", response_model=list[ScraperSetting])
+async def list_scraper_settings(
+    db: Annotated[Session, Depends(get_db)],
+) -> list[ScraperSetting]:
+    """Return all scraper_settings rows (live config for delays)."""
+    rows = (
+        db.execute(
+            text(
+                """
+            SELECT source, request_delay_sec, description, updated_at
+            FROM scraper_settings
+            ORDER BY source
+            """
+            )
+        )
+        .mappings()
+        .all()
+    )
+    return [
+        ScraperSetting(
+            source=r["source"],
+            request_delay_sec=float(r["request_delay_sec"]),
+            description=r["description"],
+            updated_at=r["updated_at"].isoformat() if r["updated_at"] else None,
+        )
+        for r in rows
+    ]
+
+
+@router.put("/scraper-settings/{source}", response_model=ScraperSetting)
+async def update_scraper_setting(
+    source: str,
+    payload: ScraperSettingUpdate,
+    db: Annotated[Session, Depends(get_db)],
+) -> ScraperSetting:
+    """Update request_delay_sec for one source. UPSERT - creates row if missing.
+
+    Invalidates the in-process cache so the change takes effect on next
+    scraper instantiation (usually within seconds).
+    """
+    row = (
+        db.execute(
+            text(
+                """
+            INSERT INTO scraper_settings (source, request_delay_sec, updated_at)
+            VALUES (:source, :delay, NOW())
+            ON CONFLICT (source) DO UPDATE
+              SET request_delay_sec = EXCLUDED.request_delay_sec,
+                  updated_at = NOW()
+            RETURNING source, request_delay_sec, description, updated_at
+            """
+            ),
+            {"source": source, "delay": payload.request_delay_sec},
+        )
+        .mappings()
+        .first()
+    )
+    db.commit()
+    invalidate_cache(source)
+    if row is None:
+        raise HTTPException(500, "Failed to upsert scraper_settings row")
+    return ScraperSetting(
+        source=row["source"],
+        request_delay_sec=float(row["request_delay_sec"]),
+        description=row["description"],
+        updated_at=row["updated_at"].isoformat() if row["updated_at"] else None,
+    )
+
+
+# -- Yandex ad-hoc scrape triggers --------------------------------------------
+
+
+class YandexDetailTriggerResp(BaseModel):
+    ok: bool
+    offer_url: str
+    offer_id: str | None
+    price_rub: int | None
+    title: str | None
+    photo_count: int
+
+
+@router.post("/scrape/yandex-detail", response_model=YandexDetailTriggerResp)
+async def scrape_yandex_detail(
+    offer_url: str,
+) -> YandexDetailTriggerResp:
+    """Ad-hoc parse one Yandex offer detail page.
+
+    Returns a snapshot of the extracted DetailEnrichment (no DB write - read-only
+    debug; main pipeline writes via estimator on /estimate flow).
+    """
+    async with YandexDetailScraper() as scraper:
+        result = await scraper.fetch_detail(offer_url)
+    if result is None:
+        raise HTTPException(404, f"Could not parse Yandex detail page: {offer_url}")
+    return YandexDetailTriggerResp(
+        ok=True,
+        offer_url=offer_url,
+        offer_id=result.offer_id,
+        price_rub=result.price_rub,
+        title=result.title,
+        photo_count=len(result.photo_urls),
+    )
+
+
+class YandexNewbuildingTriggerResp(BaseModel):
+    ok: bool
+    ext_id: str
+    ext_slug: str
+    name: str | None
+    lat: float | None
+    lon: float | None
+    rating: float | None
+    ratings_count: int | None
+    text_reviews_count: int | None
+    developer_name: str | None
+
+
+@router.post("/scrape/yandex-newbuilding", response_model=YandexNewbuildingTriggerResp)
+async def scrape_yandex_newbuilding(
+    slug: str,
+    id: str,
+    city: str = "ekaterinburg",
+) -> YandexNewbuildingTriggerResp:
+    """Ad-hoc parse a Yandex JK landing page.
+
+    URL: /{city}/kupit/novostrojka/<slug>-<id>/
+    """
+    async with YandexNewbuildingScraper() as scraper:
+        result = await scraper.fetch_jk(jk_slug=slug, jk_id=id, city=city)
+    if result is None:
+        raise HTTPException(404, f"Could not parse Yandex JK: {slug}-{id} in {city}")
+    return YandexNewbuildingTriggerResp(
+        ok=True,
+        ext_id=result.ext_id,
+        ext_slug=result.ext_slug,
+        name=result.name,
+        lat=result.lat,
+        lon=result.lon,
+        rating=result.rating,
+        ratings_count=result.ratings_count,
+        text_reviews_count=result.text_reviews_count,
+        developer_name=result.developer_name,
+    )
+
+
+class YandexValuationTriggerResp(BaseModel):
+    ok: bool
+    address: str
+    year_built: int | None
+    total_floors: int | None
+    house_type: str | None
+    has_lift: bool | None
+    total_objects: int | None
+    history_items_count: int
+
+
+@router.post("/scrape/yandex-valuation", response_model=YandexValuationTriggerResp)
+async def scrape_yandex_valuation(
+    address: str,
+    offer_category: str = "APARTMENT",
+    offer_type: str = "SELL",
+    page: int = 1,
+) -> YandexValuationTriggerResp:
+    """Ad-hoc fetch Yandex Valuation house-history for an address (debug, no cache write).
+
+    Anonymous GET - works without auth.
+    """
+    async with YandexValuationScraper() as scraper:
+        result = await scraper.fetch_house_history(
+            address=address,
+            offer_category=offer_category,
+            offer_type=offer_type,
+            page=page,
+        )
+    if result is None:
+        raise HTTPException(404, f"Could not parse Yandex Valuation: {address}")
+    return YandexValuationTriggerResp(
+        ok=True,
+        address=address,
+        year_built=result.house.year_built,
+        total_floors=result.house.total_floors,
+        house_type=result.house.house_type,
+        has_lift=result.house.has_lift,
+        total_objects=result.house.total_objects,
+        history_items_count=len(result.history_items),
+    )

tradein-mvp/backend/app/services/scraper_settings.py

@@ -0,0 +1,101 @@
+"""Runtime config loader for scraper delays.
+
+Backed by scraper_settings table with 60s in-process cache.
+"""
+
+from __future__ import annotations
+
+import logging
+import time
+from threading import Lock
+
+from sqlalchemy import text
+
+logger = logging.getLogger(__name__)
+
+# In-process cache: {source: (delay_sec, fetched_at_epoch)}
+_CACHE: dict[str, tuple[float, float]] = {}
+_CACHE_LOCK = Lock()
+_CACHE_TTL_SEC = 60.0
+
+# Default fallback if DB row missing or DB unreachable
+_DEFAULT_DELAY_BY_SOURCE: dict[str, float] = {
+    "yandex": 5.0,
+    "yandex_detail": 5.0,
+    "yandex_newbuilding": 5.0,
+    "yandex_valuation": 5.0,
+}
+_GLOBAL_DEFAULT_DELAY = 5.0
+
+# Yandex sub-scrapers all read the umbrella "yandex" key.
+# Map: scraper_name -> DB row key.
+_KEY_ALIASES: dict[str, str] = {
+    "yandex": "yandex",
+    "yandex_detail": "yandex",
+    "yandex_newbuilding": "yandex",
+    "yandex_realty_nb": "yandex",  # BaseScraper.name used by YandexNewbuildingScraper
+    "yandex_valuation": "yandex",
+    # Non-Yandex sources resolve to their own keys for forward-compat
+    # (no rows exist yet for them; fall back to default).
+}
+
+
+def _open_session() -> object:
+    """Return a new SessionLocal context-manager instance.
+
+    Deferred import avoids triggering Settings() validation at module import
+    time (unit tests run without DATABASE_URL env var).
+    Extracted as a named function so tests can patch it.
+    """
+    from app.core.db import SessionLocal
+
+    return SessionLocal()
+
+
+def get_scraper_delay(source: str) -> float:
+    """Return the configured request_delay_sec for `source`.
+
+    - Maps Yandex sub-scrapers to the umbrella 'yandex' key.
+    - Cached 60s in-process to avoid per-instance DB query.
+    - Any DB error -> returns the hardcoded default for that source.
+    """
+    key = _KEY_ALIASES.get(source, source)
+
+    now = time.time()
+    with _CACHE_LOCK:
+        cached = _CACHE.get(key)
+        if cached is not None and now - cached[1] < _CACHE_TTL_SEC:
+            return cached[0]
+
+    try:
+        with _open_session() as db:
+            row = db.execute(
+                text(
+                    "SELECT request_delay_sec FROM scraper_settings WHERE source = :s"
+                ),
+                {"s": key},
+            ).first()
+        if row is not None:
+            value = float(row[0])
+        else:
+            value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
+    except Exception as e:
+        logger.warning("scraper_settings: load failed for %s -- using default: %s", key, e)
+        value = _DEFAULT_DELAY_BY_SOURCE.get(key, _GLOBAL_DEFAULT_DELAY)
+
+    with _CACHE_LOCK:
+        _CACHE[key] = (value, now)
+    return value
+
+
+def invalidate_cache(source: str | None = None) -> None:
+    """Clear the cache entry for `source`, or the whole cache if None.
+
+    Called by admin endpoint after PUT to force a re-read on next get_scraper_delay().
+    """
+    with _CACHE_LOCK:
+        if source is None:
+            _CACHE.clear()
+        else:
+            key = _KEY_ALIASES.get(source, source)
+            _CACHE.pop(key, None)

tradein-mvp/backend/app/services/scrapers/yandex_detail.py

@@ -15,6 +15,7 @@ from typing import Any
 from pydantic import BaseModel, Field
 from selectolax.parser import HTMLParser, Node
 
+from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper
 from app.services.scrapers.yandex_helpers import (
     RE_AGENCY_FOUNDED,
@@ -103,7 +104,11 @@ class YandexDetailScraper(BaseScraper):
 
     name = "yandex_detail"
     base_url = "https://realty.yandex.ru"
-    request_delay_sec = 4.0
+    request_delay_sec = 5.0  # class default; instance value loaded from scraper_settings
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.request_delay_sec = get_scraper_delay(self.name)
 
     # BaseScraper requires fetch_around — detail isn't geo-based, raise NotImplementedError
     async def fetch_around(

tradein-mvp/backend/app/services/scrapers/yandex_newbuilding.py

@@ -14,6 +14,7 @@ from typing import Any
 from pydantic import BaseModel, Field
 from selectolax.parser import HTMLParser, Node
 
+from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper
 from app.services.scrapers.yandex_helpers import (
     RE_JK_ID,
@@ -122,7 +123,11 @@ _WORD_NUM: dict[str, int] = {
 class YandexNewbuildingScraper(BaseScraper):
     name = "yandex_realty_nb"
     base_url = "https://realty.yandex.ru"
-    request_delay_sec = 4.0
+    request_delay_sec = 5.0  # class default; instance value loaded from scraper_settings
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.request_delay_sec = get_scraper_delay(self.name)
 
     async def fetch_around(
         self, lat: float, lon: float, radius_m: int = 1000

tradein-mvp/backend/app/services/scrapers/yandex_realty.py

@@ -18,6 +18,7 @@ from typing import Any
 
 from selectolax.parser import HTMLParser, Node
 
+from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper, ScrapedLot
 from app.services.scrapers.yandex_helpers import (
     RE_FLOOR,
@@ -48,6 +49,8 @@ class YandexRealtyScraper(BaseScraper):
     def __init__(self, city: str = DEFAULT_CITY) -> None:
         super().__init__()
         self.city = city
+        # Load global Yandex delay from DB at instantiation
+        self.request_delay_sec = get_scraper_delay(self.name)
 
     async def fetch_around(
         self,

tradein-mvp/backend/app/services/scrapers/yandex_valuation.py

@@ -25,6 +25,7 @@ from urllib.parse import urlencode
 from pydantic import BaseModel, Field
 from selectolax.parser import HTMLParser
 
+from app.services.scraper_settings import get_scraper_delay
 from app.services.scrapers.base import BaseScraper
 from app.services.scrapers.yandex_helpers import (
     parse_dmy,
@@ -119,7 +120,11 @@ class YandexValuationScraper(BaseScraper):
     name = "yandex_valuation"
     base_url = "https://realty.yandex.ru"
     valuation_path = "/otsenka-kvartiry-po-adresu-onlayn/"
-    request_delay_sec = 4.0
+    request_delay_sec = 5.0  # class default; instance value loaded from scraper_settings
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.request_delay_sec = get_scraper_delay(self.name)
 
     async def fetch_around(
         self, lat: float, lon: float, radius_m: int = 1000

tradein-mvp/backend/data/sql/053_scraper_settings.sql

@@ -0,0 +1,28 @@
+-- 053_scraper_settings.sql
+-- Purpose: Per-source live-config for scraper request delays (anti-ban knob).
+--          Loaded by app at runtime; UI can update without restart.
+-- Dependencies: none (standalone settings table)
+-- Sources: User request 2026-05-23 -- Yandex admin page + global delay knob.
+
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS scraper_settings (
+    source          text PRIMARY KEY,         -- 'yandex' / 'avito' / 'cian' / 'n1' / 'yandex_detail' / ...
+    request_delay_sec numeric(5,2) NOT NULL,  -- seconds between requests (anti-ban). Range 1-60.
+    updated_at      timestamptz NOT NULL DEFAULT NOW(),
+    -- Hint about who owns this knob (UI displays, no behavior)
+    description     text
+);
+
+COMMENT ON TABLE scraper_settings IS
+  'Per-source runtime config for scrapers. Currently exposes request_delay_sec '
+  '(anti-ban delay between HTTP requests). Editable via admin UI; backend reads '
+  'with 60s in-process cache. Source key matches BaseScraper.name in code.';
+
+-- Seed: Yandex umbrella key applies to all 4 Yandex scrapers
+-- (yandex / yandex_detail / yandex_newbuilding / yandex_valuation read this key).
+INSERT INTO scraper_settings (source, request_delay_sec, description)
+VALUES ('yandex', 5.0, 'Global delay applied to all Yandex Realty scrapers (SERP, detail, newbuilding, valuation)')
+ON CONFLICT (source) DO NOTHING;
+
+COMMIT;

tradein-mvp/backend/tests/test_scraper_settings.py

@@ -0,0 +1,118 @@
+"""Tests for scraper_settings loader (cache + DB fallback + alias mapping)."""
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from app.services import scraper_settings as ss
+
+# Patch target: _open_session is a module-level wrapper so tests can mock DB
+_PATCH_TARGET = "app.services.scraper_settings._open_session"
+
+
+@pytest.fixture(autouse=True)
+def _reset_cache():
+    ss._CACHE.clear()
+    yield
+    ss._CACHE.clear()
+
+
+def _mock_session(row_value: float | None) -> MagicMock:
+    """Return a mock that behaves like SessionLocal() context-manager."""
+    session = MagicMock()
+    session.__enter__ = MagicMock(return_value=session)
+    session.__exit__ = MagicMock(return_value=False)
+    if row_value is None:
+        session.execute.return_value.first.return_value = None
+    else:
+        session.execute.return_value.first.return_value = (row_value,)
+    # _open_session() returns the session directly (no extra call)
+    mock_open = MagicMock(return_value=session)
+    return mock_open, session
+
+
+def test_returns_db_value_for_yandex():
+    mock_open, _ = _mock_session(7.5)
+    with patch(_PATCH_TARGET, mock_open):
+        assert ss.get_scraper_delay("yandex") == 7.5
+
+
+def test_yandex_subkeys_map_to_yandex_umbrella():
+    """yandex_detail / yandex_newbuilding / yandex_valuation all read 'yandex' row."""
+    mock_open, _session = _mock_session(8.0)
+    with patch(_PATCH_TARGET, mock_open):
+        assert ss.get_scraper_delay("yandex_detail") == 8.0
+        assert ss.get_scraper_delay("yandex_newbuilding") == 8.0
+        assert ss.get_scraper_delay("yandex_valuation") == 8.0
+    # All 3 lookups hit the cache after the first; _open_session called once
+    assert mock_open.call_count == 1
+
+
+def test_cache_hit_within_ttl():
+    mock_open, _session = _mock_session(6.0)
+    with patch(_PATCH_TARGET, mock_open):
+        ss.get_scraper_delay("yandex")
+        ss.get_scraper_delay("yandex")
+        ss.get_scraper_delay("yandex")
+    assert mock_open.call_count == 1  # cached after first
+
+
+def test_cache_expires_after_ttl():
+    mock_open, _session = _mock_session(6.0)
+    with patch(_PATCH_TARGET, mock_open):
+        ss.get_scraper_delay("yandex")
+        # Force-age the cache
+        with ss._CACHE_LOCK:
+            v, t = ss._CACHE["yandex"]
+            ss._CACHE["yandex"] = (v, t - 999)
+        ss.get_scraper_delay("yandex")
+    assert mock_open.call_count == 2
+
+
+def test_db_missing_row_returns_default():
+    mock_open, _ = _mock_session(None)
+    with patch(_PATCH_TARGET, mock_open):
+        assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]
+
+
+def test_db_error_returns_default():
+    bad_session = MagicMock()
+    bad_session.__enter__ = MagicMock(return_value=bad_session)
+    bad_session.__exit__ = MagicMock(return_value=False)
+    bad_session.execute.side_effect = RuntimeError("db down")
+    bad_open = MagicMock(return_value=bad_session)
+    with patch(_PATCH_TARGET, bad_open):
+        assert ss.get_scraper_delay("yandex") == ss._DEFAULT_DELAY_BY_SOURCE["yandex"]
+
+
+def test_unknown_source_uses_global_default():
+    mock_open, _ = _mock_session(None)
+    with patch(_PATCH_TARGET, mock_open):
+        assert ss.get_scraper_delay("some_new_source") == ss._GLOBAL_DEFAULT_DELAY
+
+
+def test_invalidate_cache_specific_source():
+    mock_open, _ = _mock_session(6.0)
+    with patch(_PATCH_TARGET, mock_open):
+        ss.get_scraper_delay("yandex")
+        ss.invalidate_cache("yandex")
+        ss.get_scraper_delay("yandex")
+    assert mock_open.call_count == 2
+
+
+def test_invalidate_cache_all():
+    mock_open, _ = _mock_session(6.0)
+    with patch(_PATCH_TARGET, mock_open):
+        ss.get_scraper_delay("yandex")
+        ss.invalidate_cache()
+        ss.get_scraper_delay("yandex")
+    assert mock_open.call_count == 2
+
+
+def test_invalidate_alias_invalidates_umbrella():
+    """invalidate_cache('yandex_detail') should clear the 'yandex' umbrella key."""
+    mock_open, _ = _mock_session(6.0)
+    with patch(_PATCH_TARGET, mock_open):
+        ss.get_scraper_delay("yandex_detail")
+        ss.invalidate_cache("yandex_detail")
+        ss.get_scraper_delay("yandex_detail")
+    assert mock_open.call_count == 2

tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py

@@ -0,0 +1,35 @@
+"""Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings."""
+from unittest.mock import patch
+
+from app.services.scrapers.yandex_detail import YandexDetailScraper
+from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
+from app.services.scrapers.yandex_realty import YandexRealtyScraper
+from app.services.scrapers.yandex_valuation import YandexValuationScraper
+
+
+def test_yandex_realty_picks_up_delay():
+    with patch("app.services.scrapers.yandex_realty.get_scraper_delay", return_value=9.5):
+        s = YandexRealtyScraper()
+        assert s.request_delay_sec == 9.5
+
+
+def test_yandex_detail_picks_up_delay():
+    with patch("app.services.scrapers.yandex_detail.get_scraper_delay", return_value=9.5):
+        s = YandexDetailScraper()
+        assert s.request_delay_sec == 9.5
+
+
+def test_yandex_newbuilding_picks_up_delay():
+    with patch(
+        "app.services.scrapers.yandex_newbuilding.get_scraper_delay", return_value=9.5
+    ):
+        s = YandexNewbuildingScraper()
+        assert s.request_delay_sec == 9.5
+
+
+def test_yandex_valuation_picks_up_delay():
+    with patch(
+        "app.services.scrapers.yandex_valuation.get_scraper_delay", return_value=9.5
+    ):
+        s = YandexValuationScraper()
+        assert s.request_delay_sec == 9.5