refactor(tradein/scrapers): удалить 6 осиротевших legacy scraper-модулей + тест-хирургия (#2397 Part E2)

app.services.scrapers.{cian,cian_detail,yandex_realty,domclick,domclick_exceptions,
avito_houses} имели 0 runtime-импортёров после E1 (scrape_pipeline.py удалён,
#2405) — admin.py/cian_history_backfill.py уже на scraper_kit.providers.*.

Удаление:
- 6 модулей целиком (avito_houses.py, cian.py, cian_detail.py, domclick.py,
  domclick_exceptions.py, yandex_realty.py)
- 3 осиротевших dev-sweep скрипта (local-sweep-ekb{,-cian,-yandex}.py) —
  эксклюзивно драйвили удалённые легаси-скраперы, kit-путь их заменяет

Тесты (18 файлов DELETE — эксклюзивно тестировали удалённые модули или были
чистым legacy-vs-kit parity, чей легаси-эталон исчез):
test_cian_detail.py, test_cian_serp_scraper.py, tests/scrapers/test_cian_exhaustive.py,
test_yandex_realty_serp.py, tests/scrapers/test_domclick.py,
test_avito_houses_dedup.py, test_avito_houses_parse.py,
test_houses_rating_filter_entries_1789.py, test_scraper_kit_domclick_golden_parity.py,
tests/scrapers/test_admin_{avito,cian,yandex}_kit_parity.py.

CONVERT (файл покрывает СМЕШАННЫЙ live+legacy набор — вырезаны только
секции удалённых модулей, live-покрытие сохранено):
- test_scraper_kit_cian_golden_parity.py — SERP/`_format_address`/detail
  (cian.py/cian_detail.py) убраны; extract_state (cian_state_parser, live)/
  valuation (cian_valuation, live)/newbuilding (cian_newbuilding, live)/
  no-app-imports guard — остались
- test_scraper_kit_yandex_golden_parity.py — SERP `_entity_to_lot`/
  `_parse_gate_json`/`_build_url` (yandex_realty) убраны; normalize_house_type
  (live)/newbuilding parse (yandex_newbuilding, live)/no-app-imports guard —
  остались
- test_scraper_kit_bisection_parity.py — cian/domclick/yandex-ветки убраны;
  avito-ветка (live, единственная оставшаяся легаси-бисекция) — осталась
  полностью, включая full-seed-grid
- test_scraper_kit_group_c_backfill_kit_parity.py — cian fetch_detail
  browser_fetcher parity-тест убран; avito/house_imv_backfill/
  cian_history_backfill/ekb_geoportal тесты — остались
- test_scraper_kit_pricehistory_session_parity.py — fetch_detail
  config-injection/proxy-wiring тесты (cian_detail) убраны; extract_state
  (cian_state_parser, live)/ScrapedLot (base, live) parity — остались
- test_yandex_scrapers_delay_wiring.py — yandex_realty delay-тест убран
  (kit-эквивалент уже покрыт test_kit_serp_proxy_pool.py); yandex_newbuilding
  (live)/yandex_detail/yandex_valuation (kit) тесты — остались
- test_scraper_proxy.py — yandex_realty BrowserFetcher-тест убран (redundant
  с test_kit_serp_proxy_pool.py::test_yandex_serp_no_pool_by_default)

RETARGET на kit (символ живёт, только источник поменялся):
- test_domclick_sweep.py — `_map_item` → scraper_kit.providers.domclick.serp
- test_yandex_city_sweep.py — DEFAULT_PRICE_RANGES/ROOM_PATH →
  scraper_kit.providers.yandex.serp
- test_pipeline_browser_routing.py — fetch_house_catalog →
  scraper_kit.providers.avito.houses (+ KitAvitoBlockedError для firewall-теста)
- test_snapshot_writer.py — DetailEnrichment/save_detail_enrichment →
  scraper_kit.providers.cian.detail (byte-identical strangler-копия)
- test_backfill_wave2.py — DetailEnrichment → scraper_kit.providers.cian.detail
- tests/scrapers/test_avito_unix_date_tz_consistency.py — avito_houses →
  scraper_kit.providers.avito.houses (houses-ветка _unix_to_date/RUS_MONTHS
  consolidation-проверки)

Stale-комментарии поправлены в cian_price_history.py, tests/scrapers/
test_domclick_detail.py (описывали ещё-живой domclick.py/domclick_exceptions.py).

grep -rn scrapers\.\(cian\|cian_detail\|yandex_realty\|domclick\|domclick_exceptions\|
avito_houses\) app/ scripts/ packages/ — только historical footnotes
("strangler-копия app.services.scrapers.X", past-tense "удалён/deleted").

Backend suite: 2811 passed, 6 skipped, 1 failed (known flake test_search_cache_hit
#2208, не регрессия). ruff check чист на всех изменённых файлах.

tradein-mvp/backend/app/services/cian_price_history.py

@@ -15,9 +15,10 @@ import logging
 import time
 from dataclasses import dataclass, field
 
-# #2306: fetch_detail/save_detail_enrichment migrated to scraper_kit (golden-parity
+# #2306: fetch_detail/save_detail_enrichment migrated to scraper_kit (byte-identical
-# with legacy app.services.scrapers.cian_detail, см. tests/test_scraper_kit_cian_golden_parity.py
+# golden-parity была доказана против legacy app.services.scrapers.cian_detail до его
-# + tests/test_scraper_kit_pricehistory_session_parity.py). RealScraperConfig — тот же
+# удаления, #2397 Part E2 — см. tests/test_scraper_kit_pricehistory_session_parity.py
+# для оставшейся extract_state/ScrapedLot parity). RealScraperConfig — тот же
 # read-only адаптер над settings, что и остальные kit-инжекции (#2131) — сохраняет
 # proxy-поведение (config.cian_proxy_url) идентичным прежнему прямому импорту settings.
 from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment

tradein-mvp/backend/app/services/scrapers/cian_detail.py

@@ -1,464 +0,0 @@
-"""Cian.ru detail page scraper — single offer enrichment.
-
-Different from SERP (cian.py):
-- URL: https://ekb.cian.ru/sale/flat/<offer_id>/ or https://www.cian.ru/sale/flat/<offer_id>/
-- MFE: 'frontend-offer-card'
-- State KEY: 'defaultState' (NOT 'initialState' — SERP uses initialState)
-- Sister containers in same _cianConfig: bti, priceChanges, stats, agent, newObject
-
-Parses ~88 offer fields + sister data → DetailEnrichment dataclass.
-Stage 5 of CianScraper v1.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-from dataclasses import dataclass, field
-from typing import TYPE_CHECKING, Any
-
-from curl_cffi.requests import AsyncSession
-from sqlalchemy import text
-from sqlalchemy.orm import Session
-
-from app.core.config import settings
-from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
-from app.services.scrapers.repair_state_normalizer import (
-    infer_repair_state_from_text,
-    normalize_repair_state,
-)
-from app.services.scrapers.snapshot_writer import upsert_listing_snapshot
-
-if TYPE_CHECKING:
-    from app.services.scrapers.browser_fetcher import BrowserFetcher
-
-logger = logging.getLogger(__name__)
-
-# offer_price_history.diff_percent — NUMERIC(8,2) (миграция
-# 131_fix_diff_percent_overflow.sql). Прямой INSERT ниже читает diff_percent из
-# скрапленного Cian JSON и не проходит через БД-триггер record_listing_price_change()
-# (тот срабатывает только на UPDATE listings) — гарбадж/экстремальное значение
-# с редизайна source-страницы иначе переполняет колонку.
-_DIFF_PERCENT_BOUND = 999999.99
-
-
-def _clamp_diff_percent(value: float | int | None) -> float | None:
-    """Клампит diff_percent в границы NUMERIC(8,2) (зеркало migration 131 trigger-clamp)."""
-    if value is None or isinstance(value, bool):
-        return None
-    try:
-        numeric = float(value)
-    except (TypeError, ValueError):
-        return None
-    return min(max(numeric, -_DIFF_PERCENT_BOUND), _DIFF_PERCENT_BOUND)
-
-
-@dataclass
-class DetailEnrichment:
-    """Result of cian_detail.fetch_detail() — fields для UPDATE listings + sister rows."""
-
-    # Core listing enrichment (extend existing listings row)
-    cian_id: int | None = None
-    windows_view_type: str | None = None  # 'yard' / 'street' / 'yardAndStreet'
-    separate_wcs_count: int | None = None
-    combined_wcs_count: int | None = None
-    ceiling_height: float | None = None  # meters — offer.building.ceilingHeight (#1791)
-    repair_type: str | None = None  # raw Cian: offer.repairType OR offer.decoration (#1791)
-    repair_state: str | None = None  # enum: needs_repair/standard/good/excellent
-    kitchen_area_m2: float | None = None  # offer.kitchenArea (м²)
-    description: str | None = None  # offer.description (текст объявления)
-    views_total: int | None = None  # Cian stats.totalViewsFormattedString → int
-    views_today: int | None = None
-
-    # BTI (вторичка only — primary buildings don't have БТИ)
-    bti_data: dict[str, Any] | None = None  # raw bti.houseData snapshot
-
-    # Price history (offer.priceChanges)
-    price_changes: list[dict[str, Any]] = field(default_factory=list)
-
-    # Agent profile (offer.agent.* expanded)
-    agent_profile: dict[str, Any] | None = None
-
-    # Newbuilding link (if offer is from a newbuilding)
-    newbuilding_data: dict[str, Any] | None = None
-
-    # Raw payload для debugging / future-extraction
-    raw_offer: dict[str, Any] | None = None
-    raw_sister_states: dict[str, Any] = field(default_factory=dict)
-
-
-async def fetch_detail(
-    offer_url: str,
-    *,
-    session: AsyncSession | None = None,
-    browser_fetcher: BrowserFetcher | None = None,
-) -> DetailEnrichment | None:
-    """Fetch detail page and extract all containers.
-
-    Args:
-        offer_url: e.g. 'https://ekb.cian.ru/sale/flat/327830237/'
-        session: optional shared curl_cffi session (для batched scrapes).
-            Ignored when browser_fetcher is provided.
-        browser_fetcher: optional BrowserFetcher instance (camoufox HTTP service).
-            When provided, fetches JS-rendered HTML via the browser service instead of
-            curl_cffi — enables priceChanges extraction which requires JS rendering.
-            Caller is responsible for the context-manager lifecycle of the fetcher.
-
-    Returns: DetailEnrichment, or None если fetch / parse failed.
-    """
-    if browser_fetcher is not None:
-        # Browser path: get fully JS-rendered HTML; same parse path follows.
-        try:
-            html = await browser_fetcher.fetch(offer_url)
-        except Exception as exc:
-            logger.warning("Cian detail browser fetch failed %s: %s", offer_url, exc)
-            return None
-    else:
-        # curl_cffi path (legacy, back-compat).
-        close_session = False
-        if session is None:
-            # proxies: mobile-proxy egress (#806) — без прокси datacenter-IP блокируется Cian.
-            # Пусто (env не задан) → прямое подключение (dev/no-op).
-            _proxy_url = settings.cian_proxy_url
-            _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
-            session = AsyncSession(
-                impersonate="chrome120",
-                timeout=25.0,
-                proxies=_proxies,
-                headers={
-                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                    "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
-                },
-            )
-            close_session = True
-
-        try:
-            resp = await session.get(offer_url, allow_redirects=True)
-            if resp.status_code != 200:
-                logger.warning("Cian detail fetch %s → HTTP %d", offer_url, resp.status_code)
-                return None
-            html = resp.text
-        finally:
-            if close_session:
-                await session.close()
-
-    # ── Shared parse path (identical regardless of HTML source) ──────────────
-
-    # Primary state container — defaultState in frontend-offer-card MFE
-    # NOTE: detail pages use 'defaultState', SERP uses 'initialState'
-    offer_state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
-    if offer_state is None:
-        logger.warning("Cian detail %s: defaultState extraction failed", offer_url)
-        return None
-
-    offer_data = offer_state.get("offerData", {})
-    offer = offer_data.get("offer", {})
-
-    result = DetailEnrichment(
-        cian_id=offer.get("cianId") or offer.get("id"),
-        windows_view_type=_extract_windows_view(offer),
-        separate_wcs_count=offer.get("separateWcsCount"),
-        combined_wcs_count=offer.get("combinedWcsCount"),
-        ceiling_height=_extract_ceiling_height(offer),
-        repair_type=_extract_repair_type(offer),
-        repair_state=_extract_repair_state(offer),
-        kitchen_area_m2=_parse_float(offer.get("kitchenArea")),
-        description=offer.get("description") or None,
-        raw_offer=offer,
-    )
-
-    # Stats (views) — from stats key in offerData or top-level state
-    stats = offer_data.get("stats") or offer_state.get("stats") or {}
-    result.views_total = _parse_views(stats.get("totalViewsFormattedString"))
-    result.views_today = _parse_views(stats.get("todayViewsFormattedString"))
-
-    # Price changes — browser HTML: offerData.priceChanges (priceData.price format)
-    # curl_cffi HTML: offer.priceChanges or state.priceChanges (flat price field)
-    result.price_changes = _extract_price_changes(offer, offer_data, offer_state)
-
-    # Agent profile expansion
-    result.agent_profile = _extract_agent_profile(offer)
-
-    # Sister state containers: bti, newObject, plus raw stash of everything else
-    all_states = extract_all_states(html)
-    offer_card_states = all_states.get("frontend-offer-card", {})
-
-    bti_state = offer_card_states.get("bti")
-    if bti_state:
-        result.bti_data = bti_state.get("houseData")
-        result.raw_sister_states["bti"] = bti_state
-
-    # Newbuilding link (if applicable)
-    newbuilding = offer.get("newbuilding")
-    if newbuilding and newbuilding.get("id"):
-        result.newbuilding_data = newbuilding
-
-    # Stash all non-defaultState sister containers for debugging / future-extraction
-    for k, v in offer_card_states.items():
-        if k != "defaultState":
-            result.raw_sister_states[k] = v
-
-    return result
-
-
-# ── Field extractors ──────────────────────────────────────────────────────────
-
-
-def _extract_windows_view(offer: dict[str, Any]) -> str | None:
-    return offer.get("windowsViewType")
-
-
-def _extract_ceiling_height(offer: dict[str, Any]) -> float | None:
-    """Высота потолков (м).
-
-    На detail-странице (defaultState) значение лежит в ``offer.building.ceilingHeight``,
-    а НЕ в ``offer.ceilingHeight`` (последнее всегда null) — #1791. Top-level оставлен
-    как fallback на случай иной структуры карточки.
-    """
-    building = offer.get("building") or {}
-    return _parse_float(building.get("ceilingHeight") or offer.get("ceilingHeight"))
-
-
-def _extract_repair_type(offer: dict[str, Any]) -> str | None:
-    # Cian: 'cosmetic' / 'design' / 'no' / 'euro' / ...
-    # На detail-странице структурное поле чаще приходит как `decoration`
-    # ('preFine'/'fine'/...), а `repairType` пустое (#1791) — читаем оба.
-    return offer.get("repairType") or offer.get("decoration")
-
-
-def _extract_repair_state(offer: dict[str, Any]) -> str | None:
-    """Нормализованный enum из repairType/decoration → needs_repair/standard/good/excellent.
-
-    Если структурное поле пустое — fallback на инференс из описания (#622).
-    Структурное значение всегда в приоритете.
-    """
-    state = normalize_repair_state(offer.get("repairType") or offer.get("decoration"))
-    if state is None:
-        state = infer_repair_state_from_text(offer.get("description"))
-    return state
-
-
-def _parse_float(value: Any) -> float | None:
-    if value is None:
-        return None
-    try:
-        return float(value)
-    except (ValueError, TypeError):
-        return None
-
-
-def _parse_views(formatted_str: str | None) -> int | None:
-    """Cian's stats.totalViewsFormattedString — '1 234' → 1234."""
-    if not formatted_str:
-        return None
-    try:
-        return int(str(formatted_str).replace(" ", "").replace(",", ""))
-    except (ValueError, TypeError):
-        return None
-
-
-def _extract_price_changes(
-    offer: dict[str, Any],
-    offer_data: dict[str, Any],
-    state: dict[str, Any],
-) -> list[dict[str, Any]]:
-    """Normalize priceChanges from any of the known Cian locations/formats.
-
-    Lookup priority:
-      1. offer.priceChanges — curl_cffi flat format: {changeTime, price, diffPercent}
-      2. offerData.priceChanges — browser-rendered format: {changeTime, priceData:{price}}
-      3. state.priceChanges — top-level fallback (rare)
-
-    Normalizes to: {change_time, price_rub, diff_percent}.
-    """
-    raw = (
-        offer.get("priceChanges")
-        or offer_data.get("priceChanges")
-        or state.get("priceChanges")
-        or []
-    )
-    if not isinstance(raw, list):
-        return []
-    result = []
-    for entry in raw:
-        if not isinstance(entry, dict):
-            continue
-        # price_rub: flat field (curl_cffi) OR nested priceData.price (browser)
-        price_data = entry.get("priceData") or {}
-        price_rub = entry.get("price") or entry.get("priceRub") or price_data.get("price")
-        result.append(
-            {
-                "change_time": entry.get("changeTime") or entry.get("date"),
-                "price_rub": price_rub,
-                "diff_percent": entry.get("diffPercent") or entry.get("diff_percent"),
-            }
-        )
-    return result
-
-
-def _extract_agent_profile(offer: dict[str, Any]) -> dict[str, Any] | None:
-    """offer.agent or offer.agency or offer.user — extract canonical profile shape."""
-    agent = offer.get("agent") or {}
-    user = offer.get("user") or {}
-    if not agent and not user:
-        return None
-    return {
-        "ext_agent_id": agent.get("id") or user.get("cianUserId"),
-        "company_name": agent.get("companyName") or user.get("companyName"),
-        "is_pro": agent.get("isPro") or user.get("isPro"),
-        "skills": agent.get("skills") or [],
-        "account_type": agent.get("accountType") or user.get("accountType"),
-    }
-
-
-# ── Save helpers ──────────────────────────────────────────────────────────────
-
-
-def save_detail_enrichment(db: Session, listing_id: int, enrichment: DetailEnrichment) -> None:
-    """Persist DetailEnrichment to DB:
-
-    - UPDATE listings SET windows_view_type, ceiling_height, ... WHERE id = listing_id
-    - INSERT INTO offer_price_history rows (if price_changes non-empty)
-    - INSERT INTO agents (if agent_profile) and link via agent_id_fk
-    - bti_data не сохраняется здесь — это задача Stage 6 (houses), где есть house_id
-
-    Idempotency: re-running безопасно (UPSERT / ON CONFLICT DO NOTHING semantics).
-    """
-    # Extend listings row with detail-enriched fields
-    db.execute(
-        text("""
-            UPDATE listings SET
-                windows_view_type  = COALESCE(:wvt, windows_view_type),
-                separate_wcs_count = COALESCE(:swc, separate_wcs_count),
-                combined_wcs_count = COALESCE(:cwc, combined_wcs_count),
-                ceiling_height     = COALESCE(:ch,  ceiling_height),
-                repair_type        = COALESCE(:rt,  repair_type),
-                repair_state       = COALESCE(:rs,  repair_state),
-                kitchen_area_m2    = COALESCE(CAST(:ka AS double precision), kitchen_area_m2),
-                description        = COALESCE(:descr, description),
-                views_total        = COALESCE(:vt,  views_total),
-                views_today        = COALESCE(:vd,  views_today),
-                detail_enriched_at = NOW()
-            WHERE id = CAST(:lid AS bigint)
-        """),
-        {
-            "lid": listing_id,
-            "wvt": enrichment.windows_view_type,
-            "swc": enrichment.separate_wcs_count,
-            "cwc": enrichment.combined_wcs_count,
-            "ch": enrichment.ceiling_height,
-            "rt": enrichment.repair_type,
-            "rs": enrichment.repair_state,
-            "ka": enrichment.kitchen_area_m2,
-            "descr": enrichment.description,
-            "vt": enrichment.views_total,
-            "vd": enrichment.views_today,
-        },
-    )
-
-    # Snapshot: point-in-time observation в listings_snapshots.
-    # Берём price_rub / price_per_m2 из listings строки (только что обновлённой).
-    # Используем COALESCE — на случай если listings.price_rub NULL (не должно быть, но safe).
-    _snap_row = db.execute(
-        text("""
-            SELECT price_rub, price_per_m2
-            FROM listings
-            WHERE id = CAST(:lid AS bigint)
-        """),
-        {"lid": listing_id},
-    ).fetchone()
-    _snap_written = False
-    if _snap_row is not None and _snap_row.price_rub is not None:
-        try:
-            with db.begin_nested():
-                upsert_listing_snapshot(
-                    db,
-                    listing_id=listing_id,
-                    price_rub=int(_snap_row.price_rub),
-                    price_per_m2=int(_snap_row.price_per_m2) if _snap_row.price_per_m2 else None,
-                    run_id=None,
-                    status="active",
-                )
-            _snap_written = True
-        except Exception as exc:
-            logger.warning(
-                "save_detail_enrichment: snapshot write failed listing_id=%s: %s",
-                listing_id,
-                exc,
-            )
-
-    # Price changes — INSERT new rows, de-dup by listing_id + change_time
-    for change in enrichment.price_changes:
-        if not change.get("change_time") or not change.get("price_rub"):
-            continue
-        db.execute(
-            text("""
-                INSERT INTO offer_price_history
-                    (listing_id, change_time, price_rub, diff_percent, source)
-                VALUES (
-                    CAST(:lid AS bigint),
-                    CAST(:ct AS timestamptz),
-                    CAST(:price AS numeric),
-                    CAST(:diff AS numeric),
-                    'cian'
-                )
-                ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq DO NOTHING
-            """),
-            {
-                "lid": listing_id,
-                "ct": change["change_time"],
-                "price": change["price_rub"],
-                "diff": _clamp_diff_percent(change.get("diff_percent")),
-            },
-        )
-
-    # Agent profile UPSERT — store agent row and link to listing
-    if enrichment.agent_profile and enrichment.agent_profile.get("ext_agent_id"):
-        ap = enrichment.agent_profile
-        row = db.execute(
-            text("""
-                INSERT INTO agents
-                    (ext_source, ext_agent_id, company_name, is_pro, skills, account_type)
-                VALUES (
-                    'cian',
-                    :eai,
-                    :cn,
-                    :ip,
-                    CAST(:sk AS jsonb),
-                    :at
-                )
-                ON CONFLICT (ext_source, ext_agent_id) DO UPDATE SET
-                    company_name = COALESCE(EXCLUDED.company_name, agents.company_name),
-                    is_pro       = COALESCE(EXCLUDED.is_pro,       agents.is_pro),
-                    skills       = COALESCE(EXCLUDED.skills,       agents.skills),
-                    account_type = COALESCE(EXCLUDED.account_type, agents.account_type)
-                RETURNING id
-            """),
-            {
-                "eai": str(ap["ext_agent_id"]),
-                "cn": ap.get("company_name"),
-                "ip": ap.get("is_pro"),
-                "sk": json.dumps(ap.get("skills") or []),
-                "at": ap.get("account_type"),
-            },
-        )
-        agent_db_id = row.scalar_one_or_none()
-        if agent_db_id is not None:
-            db.execute(
-                text("""
-                    UPDATE listings
-                    SET agent_id_fk = CAST(:aid AS bigint)
-                    WHERE id = CAST(:lid AS bigint)
-                      AND agent_id_fk IS NULL
-                """),
-                {"aid": agent_db_id, "lid": listing_id},
-            )
-
-    db.commit()
-    logger.info(
-        "Cian detail enrichment saved for listing_id=%s (price_changes=%d, agent=%s, snapshot=%s)",
-        listing_id,
-        len(enrichment.price_changes),
-        bool(enrichment.agent_profile),
-        _snap_written,
-    )

tradein-mvp/backend/app/services/scrapers/domclick.py

@@ -1,667 +0,0 @@
-"""DomClick.ru scraper — вторичка через JSON BFF API (#1968).
-
-Стратегия: GET https://bff-search-web.domclick.ru/api/offers/v1?... через
-BrowserFetcher(source="domclick") (generic provider → shared mobile proxy).
-QRATOR банит прямые datacenter-запросы, но пропускает через mobile proxy.
-
-Ответ BrowserFetcher содержит JSON, обёрнутый в HTML (<pre> или bare body).
-Парсинг: _extract_json() вытаскивает первый {...} из ответа.
-
-Стратегия нумерации комнат (ROOM_BUCKETS):
-  "st"  — студии (force rooms=0 в ScrapedLot)
-  "1"–"4" — соответственно
-  "5+"  — 5 и более комнат
-
-Пагинация: offset 0, 20, 40, …, cap=2000. Если snippetsCount > 2000
-для бакета, рекурсивно делим по цене (binary split) до тех пор пока
-каждая ветка укладывается в cap.
-
-# TODO Layer B (#1846 follow-up): backfill renovation/wallType/priceHistory
-через detail-эндпоинт после первоначального сбора.
-"""
-
-from __future__ import annotations
-
-import hashlib
-import json
-import logging
-from datetime import date
-from typing import Any
-from urllib.parse import urlencode
-
-from app.services.scraper_settings import get_scraper_delay
-from app.services.scrapers.base import BaseScraper, ScrapedLot
-from app.services.scrapers.domclick_exceptions import DomClickBlockedError
-from app.services.scrapers.repair_state_normalizer import infer_repair_state_from_text
-
-logger = logging.getLogger(__name__)
-
-# ── API constants ─────────────────────────────────────────────────────────────
-
-_BFF_BASE = "https://bff-search-web.domclick.ru"
-_EKB_ADDRESS_GUID = "0d475b79-88de-4054-818c-37d8f9d0d440"
-_EKB_AREA_ID = "20561"
-
-# Buckets to sweep — порядок влияет на логи.
-ROOM_BUCKETS: tuple[str, ...] = ("st", "1", "2", "3", "4", "5+")
-
-OFFSET_CAP: int = 2000  # max offset принятый BFF API
-PAGE_SIZE: int = 20  # items per page (жёстко задан API)
-# Потолок цены для binary split. 1 млрд ₽ — безопасно выше любой ЕКБ-квартиры;
-# выбран так чтобы НЕ отбрасывать листинги дороже потолка при сплите unbounded-бакета
-# (родительский count не ограничен сверху → clamped _lte должен покрывать весь хвост).
-LTE_MAX: int = 1_000_000_000
-MIN_PRICE_SPAN: int = 100_000  # ниже этого span прекращаем делить
-
-# ── QRATOR block detection ────────────────────────────────────────────────────
-
-_QRATOR_MARKERS: tuple[str, ...] = (
-    "qrator",
-    "bot_mitigation",
-    "система защиты",
-    "403 | домклик",
-    "captcha",
-    "access denied",
-)
-
-# ── EKB geo guard ─────────────────────────────────────────────────────────────
-
-_EKB_LAT_MIN: float = 56.6
-_EKB_LAT_MAX: float = 57.0
-_EKB_LON_MIN: float = 60.2
-_EKB_LON_MAX: float = 60.9
-_EKB_REGION_NAME: str = "Екатеринбург"
-
-
-# ── JSON extraction ───────────────────────────────────────────────────────────
-
-
-def _extract_json(html: str) -> dict[str, Any]:
-    """Извлекает JSON-объект из ответа BrowserFetcher.
-
-    BrowserFetcher оборачивает JSON в HTML-страницу двумя способами:
-    1. Bare body: ``{"result": ...}`` напрямую в <body>.
-    2. <pre>-wrapped: ``<html><body><pre>{"result": ...}</pre></body></html>``.
-
-    Стратегия: найти первый '{' и последний '}' — работает для обоих вариантов.
-
-    Raises:
-        DomClickBlockedError: если ответ содержит QRATOR/captcha маркеры.
-        ValueError: если JSON не найден или не является dict.
-    """
-    # Сканируем ВЕСЬ ответ (а не только первые 4096B): block-маркер может
-    # стоять за пределами head в крупных challenge-страницах.
-    html_lower = html.lower()
-    if any(m in html_lower for m in _QRATOR_MARKERS):
-        raise DomClickBlockedError(
-            f"DomClick BFF: QRATOR block page detected (markers checked: {_QRATOR_MARKERS[:2]})"
-        )
-
-    start = html.find("{")
-    end = html.rfind("}")
-    if start == -1 or end == -1 or end <= start:
-        raise ValueError(f"No JSON object found in BFF response (len={len(html)})")
-
-    data = json.loads(html[start : end + 1])
-    if not isinstance(data, dict):
-        raise ValueError(f"BFF response JSON is not a dict: {type(data)}")
-    return data  # type: ignore[return-value]
-
-
-# ── URL builders ──────────────────────────────────────────────────────────────
-
-
-def _build_offers_url(
-    rooms: str,
-    price_gte: int | None,
-    price_lte: int | None,
-    offset: int,
-) -> str:
-    """Строит URL для GET /api/offers/v1 с пагинацией.
-
-    urlencode кодирует "5+" → "5%2B" (literal '+' в query string = space → reject).
-    """
-    params: list[tuple[str, str]] = [
-        ("address", _EKB_ADDRESS_GUID),
-        ("aids", _EKB_AREA_ID),
-        ("deal_type", "sale"),
-        ("category", "living"),
-        ("offer_type", "flat"),
-        ("rooms", rooms),
-        ("sort", "qi"),
-        ("sort_dir", "desc"),
-        ("offset", str(offset)),
-        ("limit", str(PAGE_SIZE)),
-    ]
-    if price_gte is not None:
-        params.append(("sale_price__gte", str(price_gte)))
-    if price_lte is not None:
-        params.append(("sale_price__lte", str(price_lte)))
-    return f"{_BFF_BASE}/api/offers/v1?{urlencode(params)}"
-
-
-def _build_count_url(
-    rooms: str,
-    price_gte: int | None,
-    price_lte: int | None,
-) -> str:
-    """Строит URL для GET /api/offers/count/v1 (без offset/limit)."""
-    params: list[tuple[str, str]] = [
-        ("address", _EKB_ADDRESS_GUID),
-        ("aids", _EKB_AREA_ID),
-        ("deal_type", "sale"),
-        ("category", "living"),
-        ("offer_type", "flat"),
-        ("rooms", rooms),
-        ("sort", "qi"),
-        ("sort_dir", "desc"),
-    ]
-    if price_gte is not None:
-        params.append(("sale_price__gte", str(price_gte)))
-    if price_lte is not None:
-        params.append(("sale_price__lte", str(price_lte)))
-    return f"{_BFF_BASE}/api/offers/count/v1?{urlencode(params)}"
-
-
-def _parse_publish_date(iso: str | None) -> date | None:
-    """Парсит ISO8601 дату публикации → date. None при ошибке."""
-    if not iso:
-        return None
-    try:
-        return date.fromisoformat(iso[:10])
-    except (ValueError, TypeError):
-        return None
-
-
-def _extract_agency_name(seller: dict[str, Any]) -> str | None:
-    """Извлечь читаемое имя агентства/агента из seller-блока BFF offer.
-
-    seller.company / seller.agent могут быть строкой ИЛИ dict (defensive —
-    реальная форма поля варьируется). Приоритет: company → agent. Для dict
-    пробуем типичные name-поля. Возвращает None если ничего читаемого нет.
-    """
-    for key in ("company", "agent"):
-        val = seller.get(key)
-        if isinstance(val, str):
-            name = val.strip()
-            if name:
-                return name
-        elif isinstance(val, dict):
-            for name_key in ("name", "title", "fullName"):
-                nm = val.get(name_key)
-                if isinstance(nm, str) and nm.strip():
-                    return nm.strip()
-    return None
-
-
-# ── DomClickScraper ──────────────────────────────────────────────────────────
-
-
-class DomClickScraper(BaseScraper):
-    """DomClick вторичка через JSON BFF API. Источник = 'domklik'.
-
-    Использует BrowserFetcher(source="domclick") (generic provider → shared mobile
-    proxy), который обходит QRATOR. Прямые curl/httpx-запросы с datacenter-IP
-    блокируются QRATOR.
-
-    Основной метод: fetch_city(city_id, rooms, pages).
-    fetch_around() не реализован: DomClick не поддерживает geo-radius в URL.
-
-    Counters (публичные после fetch_city):
-        parse_failures  — офферы с ошибкой маппинга
-        geo_filtered    — офферы вне ЕКБ bbox или с неверным offerRegionName
-        blocked         — True если sweep был прерван QRATOR-блоком
-        fetch_errors    — не-block ошибки извлечения JSON (truncated/garbled/bad shape)
-    """
-
-    name = "domklik"
-    source = "domklik"
-    base_url = "https://domclick.ru"
-    # Консервативная задержка между страницами (fallback до init)
-    request_delay_sec = 8.0
-
-    def __init__(self) -> None:
-        super().__init__()
-        self.request_delay_sec = get_scraper_delay(self.name)
-        self.parse_failures: int = 0
-        self.geo_filtered: int = 0
-        self.blocked: bool = False
-        # Не-block ошибки извлечения JSON (truncated/garbled response, неверная
-        # структура). В отличие от parse_failures (per-item), это per-fetch ошибки,
-        # которые ограничивают сбор бакета. Учитываются в honest-status pipeline.
-        self.fetch_errors: int = 0
-
-    async def __aenter__(self) -> DomClickScraper:
-        await super().__aenter__()
-        return self
-
-    # ── fetch_around — stub ───────────────────────────────────────────────────
-
-    async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list[ScrapedLot]:
-        """DomClick не поддерживает geo-radius. Используй fetch_city()."""
-        raise NotImplementedError(
-            "DomClick не поддерживает geo-radius; используй fetch_city(city_id=...)"
-        )
-
-    # ── Основной метод ────────────────────────────────────────────────────────
-
-    async def fetch_city(
-        self,
-        city_id: int,
-        rooms: list[int] | None = None,
-        pages: int = 100,
-    ) -> list[ScrapedLot]:
-        """Citywide sweep через BFF JSON API.
-
-        Аргументы city_id и rooms принимаются для совместимости сигнатуры с
-        вызывающим кодом (run_domclick_city_sweep), но:
-        - city_id — vestigial (EКБ захардкожен через GUID).
-        - rooms — игнорируется; всегда обходятся все ROOM_BUCKETS внутри.
-
-        Args:
-            city_id: игнорируется (EKB захардкожен).
-            rooms: игнорируется (ROOM_BUCKETS перебирается всегда).
-            pages: максимальное число страниц на бакет (safety cap).
-
-        Returns:
-            Дедуплицированный по source_id список ScrapedLot.
-        """
-        # TODO Layer B (#1846 follow-up): backfill renovation/wallType/priceHistory
-
-        from app.services.scrapers.browser_fetcher import BrowserFetcher
-
-        out_lots: list[ScrapedLot] = []
-        seen_ids: set[str] = set()
-
-        async with BrowserFetcher(source="domclick") as fetcher:
-            for bucket in ROOM_BUCKETS:
-                logger.info(
-                    "domklik: BFF sweep rooms=%r city_id=%d pages_cap=%d",
-                    bucket,
-                    city_id,
-                    pages,
-                )
-                try:
-                    await self._sweep_bucket(
-                        fetcher=fetcher,
-                        rooms=bucket,
-                        price_gte=None,
-                        price_lte=None,
-                        seen_ids=seen_ids,
-                        out_lots=out_lots,
-                        pages=pages,
-                    )
-                except DomClickBlockedError:
-                    self.blocked = True
-                    logger.error(
-                        "domklik: QRATOR block during rooms=%r — aborting all buckets",
-                        bucket,
-                    )
-                    break
-                except (ValueError, TypeError) as exc:
-                    # Defensive: bucket-level ошибка не должна убивать весь sweep.
-                    # _count/_paginate уже глотают эти ошибки per-fetch (fetch_errors++),
-                    # но если что-то всё же всплыло — переходим к следующему бакету.
-                    self.fetch_errors += 1
-                    logger.warning(
-                        "domklik: bucket rooms=%r failed (%s) — skipping to next bucket",
-                        bucket,
-                        exc,
-                        exc_info=True,
-                    )
-                    continue
-
-        logger.info(
-            "domklik: fetch_city done city_id=%d total=%d "
-            "parse_failures=%d geo_filtered=%d fetch_errors=%d blocked=%s",
-            city_id,
-            len(out_lots),
-            self.parse_failures,
-            self.geo_filtered,
-            self.fetch_errors,
-            self.blocked,
-        )
-        return out_lots
-
-    # ── Internal sweep helpers ────────────────────────────────────────────────
-
-    async def _count(
-        self,
-        fetcher: Any,
-        rooms: str,
-        price_gte: int | None,
-        price_lte: int | None,
-    ) -> int:
-        """Запрашивает snippetsCount для данного бакета (rooms + price range).
-
-        DomClickBlockedError пробрасывается наверх (abort sweep). Прочие ошибки
-        извлечения JSON (truncated/garbled/bad shape) → fetch_errors++ и return 0
-        (бакет пропускается, sweep продолжается).
-        """
-        url = _build_count_url(rooms, price_gte, price_lte)
-        logger.debug("domklik: count url=%s", url)
-        html = await fetcher.fetch(url)
-        try:
-            data = _extract_json(html)
-        except DomClickBlockedError:
-            raise
-        except (ValueError, TypeError, AttributeError):
-            self.fetch_errors += 1
-            logger.warning(
-                "domklik: _count JSON extract failed rooms=%r price=[%s,%s] — skip bucket",
-                rooms,
-                price_gte,
-                price_lte,
-                exc_info=True,
-            )
-            return 0
-        # {"result": null} → None, {"snippetsCount": null} → None: harden обе.
-        res = data.get("result") or {}
-        raw = res.get("snippetsCount")
-        return int(raw) if raw else 0
-
-    async def _sweep_bucket(
-        self,
-        fetcher: Any,
-        rooms: str,
-        price_gte: int | None,
-        price_lte: int | None,
-        seen_ids: set[str],
-        out_lots: list[ScrapedLot],
-        pages: int,
-    ) -> None:
-        """Рекурсивный sweep бакета (rooms, price_gte, price_lte).
-
-        Если snippetsCount > OFFSET_CAP — делим диапазон цен пополам и
-        рекурсируем в каждую половину. Остановка рекурсии:
-        - span <= MIN_PRICE_SPAN → пагинируем как есть (с предупреждением о truncation)
-        - snippetsCount == 0 → пропускаем
-        - snippetsCount <= OFFSET_CAP → пагинируем
-
-        Raises:
-            DomClickBlockedError: если QRATOR-блок — propagate наверх.
-        """
-        count = await self._count(fetcher, rooms, price_gte, price_lte)
-        if count == 0:
-            return
-
-        if count > OFFSET_CAP:
-            _gte = price_gte if price_gte is not None else 0
-            _lte = price_lte if price_lte is not None else LTE_MAX
-            span = _lte - _gte
-            if span <= MIN_PRICE_SPAN:
-                logger.warning(
-                    "domklik: rooms=%r price=[%s,%s] count=%d > cap=%d "
-                    "but span=%d <= min=%d — paginating as-is (bucket truncated at %d)",
-                    rooms,
-                    price_gte,
-                    price_lte,
-                    count,
-                    OFFSET_CAP,
-                    span,
-                    MIN_PRICE_SPAN,
-                    OFFSET_CAP,
-                )
-                await self._paginate(
-                    fetcher, rooms, price_gte, price_lte, seen_ids, out_lots, pages
-                )
-                return
-            mid = (_gte + _lte) // 2
-            logger.debug(
-                "domklik: rooms=%r count=%d > cap=%d — price-split [%d,%d] → [%d,%d]+[%d,%d]",
-                rooms,
-                count,
-                OFFSET_CAP,
-                _gte,
-                _lte,
-                _gte,
-                mid,
-                mid + 1,
-                _lte,
-            )
-            await self._sweep_bucket(fetcher, rooms, _gte, mid, seen_ids, out_lots, pages)
-            await self._sweep_bucket(fetcher, rooms, mid + 1, _lte, seen_ids, out_lots, pages)
-        else:
-            await self._paginate(fetcher, rooms, price_gte, price_lte, seen_ids, out_lots, pages)
-
-    async def _paginate(
-        self,
-        fetcher: Any,
-        rooms: str,
-        price_gte: int | None,
-        price_lte: int | None,
-        seen_ids: set[str],
-        out_lots: list[ScrapedLot],
-        pages: int,
-    ) -> None:
-        """Пагинирует один бакет (rooms, price range) до исчерпания или cap.
-
-        DomClickBlockedError пробрасывается наверх (abort sweep). Прочие ошибки
-        извлечения JSON (truncated/garbled/bad shape) → fetch_errors++ и break
-        (трактуем как конец бакета — уже собранные lots сохраняются).
-
-        Raises:
-            DomClickBlockedError: propagate из _extract_json при QRATOR-блоке.
-        """
-        force_rooms = 0 if rooms == "st" else None
-        page_idx = 0
-        while page_idx < pages:
-            offset = page_idx * PAGE_SIZE
-            if offset >= OFFSET_CAP:
-                break
-            url = _build_offers_url(rooms, price_gte, price_lte, offset)
-            logger.debug("domklik: offers url=%s", url)
-            html = await fetcher.fetch(url)
-            try:
-                data = _extract_json(html)
-            except DomClickBlockedError:
-                raise
-            except (ValueError, TypeError, AttributeError):
-                self.fetch_errors += 1
-                logger.warning(
-                    "domklik: _paginate JSON extract failed rooms=%r offset=%d "
-                    "— ending bucket (lots so far preserved)",
-                    rooms,
-                    offset,
-                    exc_info=True,
-                )
-                break
-
-            # {"result": null} → None; harden до .get("items").
-            items: list[dict[str, Any]] = (data.get("result") or {}).get("items") or []
-            if not items:
-                break
-
-            for item in items:
-                lot = self._map_item(item, force_rooms=force_rooms)
-                if lot is None:
-                    continue
-                if not self._is_geo_ok(item):
-                    self.geo_filtered += 1
-                    continue
-                key = lot.source_id or lot.source_url
-                if key in seen_ids:
-                    continue
-                seen_ids.add(key)
-                out_lots.append(lot)
-
-            page_idx += 1
-            if page_idx < pages and len(items) == PAGE_SIZE:
-                await self.sleep_between_requests()
-            elif not items or len(items) < PAGE_SIZE:
-                break
-
-    def _is_geo_ok(self, item: dict[str, Any]) -> bool:
-        """Гео-гард: пропускает только листинги ЕКБ в bbox.
-
-        aids=20561 даёт чистый ЕКБ, но гард оставляем как defensive проверку.
-        """
-        region = item.get("offerRegionName", "")
-        if region != _EKB_REGION_NAME:
-            return False
-        loc = item.get("location") or {}
-        lat = loc.get("lat")
-        lon = loc.get("lon")
-        if lat is None or lon is None:
-            return False
-        return (
-            _EKB_LAT_MIN <= float(lat) <= _EKB_LAT_MAX
-            and _EKB_LON_MIN <= float(lon) <= _EKB_LON_MAX
-        )
-
-    def _map_item(
-        self, item: dict[str, Any], *, force_rooms: int | None = None
-    ) -> ScrapedLot | None:
-        """Маппинг offer-item из BFF API → ScrapedLot.
-
-        Возвращает None если price <= 0 или при ошибке парсинга.
-        """
-        try:
-            price = item.get("price", 0) or 0
-            if price <= 0:
-                return None
-
-            item_id = item.get("id")
-            source_id = str(item_id) if item_id is not None else None
-            source_url: str = item.get("path", "") or ""
-
-            loc = item.get("location") or {}
-            lat: float | None = loc.get("lat")
-            lon: float | None = loc.get("lon")
-
-            address_obj = item.get("address") or {}
-            address: str | None = address_obj.get("displayName")
-
-            # 0 — sentinel «неизвестно» для area/floor/total_floors/buildYear
-            # (DomClick эмитит 0 для under-construction / unknown) → None.
-            obj_info = item.get("objectInfo") or {}
-            area_raw = obj_info.get("area")
-            area_m2: float | None = float(area_raw) if area_raw else None
-            floor_raw = obj_info.get("floor")
-            floor: int | None = int(floor_raw) if floor_raw else None
-
-            house = item.get("house") or {}
-            floors_raw = house.get("floors")
-            total_floors: int | None = int(floors_raw) if floors_raw else None
-            year_raw = house.get("buildYear")
-            year_built: int | None = int(year_raw) if year_raw else None
-
-            # rooms: 0 — ВАЛИДНОЕ значение (студия), поэтому проверка is not None,
-            # а не truthy. force_rooms=0 (st-бакет) перебивает objectInfo.
-            if force_rooms is not None:
-                rooms: int | None = force_rooms
-            else:
-                rooms_raw = obj_info.get("rooms")
-                rooms = int(rooms_raw) if rooms_raw is not None else None
-
-            square_price_raw = item.get("squarePrice")
-            price_per_m2: int | None = int(square_price_raw) if square_price_raw else None
-            if price_per_m2 is None and area_m2 and area_m2 > 0:
-                price_per_m2 = int(price / area_m2)
-
-            # ── Описание + repair-инференс + minhash (Layer A enrichment) ─────
-            # DomClick search не отдаёт renovation-enum: repair_state выводим из
-            # текста описания (как cian #622 fallback — «евроремонт»/«без отделки»).
-            # description_minhash — sha1 от описания (mirror cian) для cross-source
-            # дедупа.
-            description_raw = item.get("description")
-            description: str | None = None
-            if isinstance(description_raw, str):
-                description = description_raw.strip() or None
-
-            repair_state: str | None = None
-            description_minhash: str | None = None
-            if description:
-                repair_state = infer_repair_state_from_text(description)
-                description_minhash = hashlib.sha1(
-                    description.lower().encode("utf-8", errors="replace")
-                ).hexdigest()[:32]
-
-            # ── Продавец / агентство ──────────────────────────────────────────
-            seller = item.get("seller")
-            if not isinstance(seller, dict):
-                seller = {}
-            agency_name = _extract_agency_name(seller)
-
-            flat_complex = item.get("flatComplex") or {}
-            raw_payload: dict[str, Any] = {
-                "isRosreestrApproved": item.get("isRosreestrApproved"),
-                "squarePrice": square_price_raw,
-                "lastPriceHistoryState": item.get("lastPriceHistoryState"),
-                "flatComplex": {
-                    "name": flat_complex.get("name"),
-                    "id": flat_complex.get("id"),
-                    "slug": flat_complex.get("slug"),
-                }
-                if flat_complex
-                else None,
-                "updatedDate": item.get("updatedDate"),
-            }
-
-            # ── Доп. SERP-поля (Layer A enrichment) — мержим в raw_payload, ────
-            # сбрасывая None-значения чтобы payload оставался компактным; ничего
-            # из существующих ключей не теряем.
-            seller_compact = {
-                k: seller.get(k) for k in ("agent", "company") if seller.get(k) is not None
-            }
-            _enrich: dict[str, Any] = {
-                "is_sber_collateral": item.get("isSberCollateral"),
-                "has_discount": item.get("hasDiscount"),
-                "discount_value": item.get("discountValue"),
-                "duplicates_offer_count": item.get("duplicatesOfferCount"),
-                "seller": seller_compact or None,
-            }
-            raw_payload.update({k: v for k, v in _enrich.items() if v is not None})
-
-            publish_date = _parse_publish_date(item.get("publishedDate"))
-
-            return ScrapedLot(
-                source="domklik",
-                source_url=source_url,
-                source_id=source_id,
-                address=address,
-                lat=lat,
-                lon=lon,
-                rooms=rooms,
-                area_m2=area_m2,
-                floor=floor,
-                total_floors=total_floors,
-                year_built=year_built,
-                price_rub=int(price),
-                price_per_m2=price_per_m2,
-                listing_segment="vtorichka",
-                publish_date=publish_date,
-                description=description,
-                repair_state=repair_state,
-                description_minhash=description_minhash,
-                agency_name=agency_name,
-                raw_payload=raw_payload,
-            )
-        except Exception:
-            self.parse_failures += 1
-            logger.warning(
-                "domklik: _map_item failed for item id=%r (parse_failures=%d)",
-                item.get("id"),
-                self.parse_failures,
-                exc_info=True,
-            )
-            return None
-
-
-# ── Convenience runner (для Celery tasks) ────────────────────────────────────
-
-
-async def scrape_domclick_city(
-    city_id: int,
-    rooms: list[int] | None = None,
-    pages: int = 100,
-) -> list[ScrapedLot]:
-    """Удобная точка входа для вызова из Celery tasks.
-
-    Пример::
-
-        import asyncio
-        lots = asyncio.run(scrape_domclick_city(city_id=4, rooms=None))
-    """
-    async with DomClickScraper() as scraper:
-        return await scraper.fetch_city(city_id=city_id, rooms=rooms, pages=pages)

tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py

@@ -1,27 +0,0 @@
-"""DomClick-specific exceptions для anti-bot detection."""
-
-
-class DomClickBlockedError(Exception):
-    """DomClick BFF вернул QRATOR block-страницу (HTTP 200 + block HTML).
-
-    QRATOR (qrator.net) — WAF/DDoS-защита domclick.ru. Блокирует datacenter-IP
-    и возвращает HTML с маркерами: "qrator", "bot_mitigation", "система защиты",
-    "403 | домклик", "captcha", "access denied".
-
-    Обходится через shared mobile proxy (BrowserFetcher(source="domclick") →
-    generic provider → мобильный egress).
-    """
-
-
-class DomClickParseError(Exception):
-    """DomClick detail-карточка получена (HTTP 200, не block), но SSR-стейт
-    не извлекается / не парсится.
-
-    Возникает в Layer B (domclick_detail.py) когда:
-    - `window.__SSR_STATE__` не найден в HTML (но это и не challenge-страница);
-    - object-литерал не сбалансирован по скобкам;
-    - json.loads падает после санитайза bare `undefined` → `null`.
-
-    В отличие от DomClickBlockedError (anti-bot), это сигнал дрейфа схемы карточки
-    или повреждённого ответа — оркестратор считает его parse-failure, а не блоком.
-    """

tradein-mvp/backend/tests/scrapers/test_admin_avito_kit_parity.py

@@ -1,112 +0,0 @@
-"""Parity-proof: admin.py Avito debug-роуты (#2305, Group A) — legacy vs scraper_kit.
-
-`app/api/v1/admin.py::scrape_avito_house` теперь зовёт `scraper_kit.providers.avito.houses`
-вместо `app.services.scrapers.avito_houses` (тот же `/scrape` root-роут —
-`scraper_kit.providers.avito.serp.AvitoScraper`). Каждая пара функций ниже — чистое,
-offline-тестируемое parsing-ядро, которое реально используется соответствующим
-debug-эндпоинтом (не сам HTTP-fetch — см. `tests/support/README.md`: "Live network/DB
-в parity-тестах избегайте").
-
-- `_parse_house_page` — `scrape_avito_house` → `fetch_house_catalog` парсит виджет
-  `housePage` этой функцией.
-- `_extract_rooms_from_title` / `_extract_area_from_title` — использует
-  `AvitoScraper` (SERP-парсинг `/scrape` root-роута) для title-fallback полей.
-
-Легаси `app.services.scrapers.avito_imv` (`scrape_avito_imv` → `evaluate_via_imv` →
-`_parse_price`) удалён (#2277 финальный шаг scraper_kit-миграции, 0 runtime
-importers) — parity-проверка `_parse_price` убрана вместе с ним; kit-сторона
-`_parse_price` покрыта отдельно (`tests/scrapers/test_avito_imv_kit_parity.py` /
-`tests/test_avito_imv_parse.py`).
-"""
-
-from __future__ import annotations
-
-import dataclasses
-import os
-
-# Соседние модули app.services.scrapers.* импортируют app.core.config при импорте
-# пакета — фиктивный DSN мостит офлайн-запуск (mirror test_avito_detail_kit_parity.py).
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from scraper_kit.providers.avito.houses import _parse_house_page as kit_parse_house_page
-from scraper_kit.providers.avito.serp import (
-    _extract_area_from_title as kit_extract_area_from_title,
-)
-from scraper_kit.providers.avito.serp import (
-    _extract_rooms_from_title as kit_extract_rooms_from_title,
-)
-
-from app.services.scrapers.avito import _extract_area_from_title as legacy_extract_area_from_title
-from app.services.scrapers.avito import (
-    _extract_rooms_from_title as legacy_extract_rooms_from_title,
-)
-from app.services.scrapers.avito_houses import _parse_house_page as legacy_parse_house_page
-from tests.support.parity import assert_parity
-
-_HOUSE_WIDGET = {
-    "props": {
-        "developmentData": {
-            "avitoId": 3171365,
-            "id": "abc123hash",
-            "title": "ЖК Тестовый",
-            "address": "ул. Тестовая, 1",
-            "fullAddress": "Екатеринбург, ул. Тестовая, 1",
-            "coords": {"lat": 56.8, "lng": 60.6},
-            "aboutDevelopment": {
-                "expandParams": {
-                    "items": [
-                        {
-                            "params": [
-                                {"type": "Год постройки", "value": "2020"},
-                                {"type": "Этажей", "value": "16"},
-                                {"type": "Пассажирский лифт", "value": "2"},
-                            ]
-                        }
-                    ]
-                }
-            },
-            "ratingBadge": {"info": {"score": 4.5, "scoreString": "4.5"}},
-            "ratingSummaryStat": {"reviewCount": 10, "ratingStat": []},
-            "mapPreview": {"objects": "школа рядом", "distance": "5 мин", "pins": []},
-            "developer": {"name": "ООО Застройщик", "key": "dev1"},
-        }
-    }
-}
-
-
-def test_avito_house_page_parse_are_class_distinct() -> None:
-    """Sanity: legacy/kit HouseInfo — разные классы, обычный `==` даёт False."""
-    legacy_result = legacy_parse_house_page(_HOUSE_WIDGET)
-    kit_result = kit_parse_house_page(_HOUSE_WIDGET)
-
-    assert type(legacy_result) is not type(kit_result)
-    assert legacy_result != kit_result
-    assert dataclasses.asdict(legacy_result) == dataclasses.asdict(kit_result)
-
-
-def test_parity_avito_house_page() -> None:
-    """`scrape_avito_house` (admin.py) → fetch_house_catalog → _parse_house_page."""
-    assert_parity(
-        legacy_fn=legacy_parse_house_page,
-        kit_fn=kit_parse_house_page,
-        fixtures=[(_HOUSE_WIDGET,)],
-    )
-
-
-def test_parity_avito_serp_title_extract() -> None:
-    """`/scrape` root-роут (admin.py) → AvitoScraper SERP title-fallback fields."""
-    titles = [
-        "2-к. квартира, 54.3 м², 3/9 эт.",
-        "Студия, 25 м², 1/5 эт.",
-        "3-к. кв., 75,5 м², 12/16 эт.",
-    ]
-    assert_parity(
-        legacy_fn=legacy_extract_rooms_from_title,
-        kit_fn=kit_extract_rooms_from_title,
-        fixtures=titles,
-    )
-    assert_parity(
-        legacy_fn=legacy_extract_area_from_title,
-        kit_fn=kit_extract_area_from_title,
-        fixtures=titles,
-    )

tradein-mvp/backend/tests/scrapers/test_admin_cian_kit_parity.py

@@ -1,51 +0,0 @@
-"""Parity-proof: admin.py `scrape_cian_detail` (#2305, Group A) — legacy vs scraper_kit.
-
-`app/api/v1/admin.py::scrape_cian_detail` теперь зовёт
-`scraper_kit.providers.cian.detail.fetch_detail/save_detail_enrichment` вместо
-`app.services.scrapers.cian_detail`. `_extract_price_changes` — чистая,
-offline-тестируемая normalize-функция, которую `fetch_detail` вызывает при
-сборке `DetailEnrichment.price_changes` (см. `tests/support/README.md`:
-"Live network/DB в parity-тестах избегайте").
-"""
-
-from __future__ import annotations
-
-import os
-
-# cian_detail.py импортирует app.core.config при module-level import — фиктивный
-# DSN мостит офлайн-запуск (mirror test_avito_detail_kit_parity.py).
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from scraper_kit.providers.cian.detail import _extract_price_changes as kit_extract_price_changes
-
-from app.services.scrapers.cian_detail import (
-    _extract_price_changes as legacy_extract_price_changes,
-)
-from tests.support.parity import assert_parity
-
-# Три реальных формы priceChanges: curl_cffi flat, browser-rendered nested, оба-пусто.
-_OFFER_FLAT = {
-    "priceChanges": [
-        {"changeTime": "2026-01-15T00:00:00", "price": 9000000, "diffPercent": -2.5},
-        {"changeTime": "2026-02-01T00:00:00", "priceRub": 8800000, "diff_percent": -2.2},
-    ]
-}
-_OFFER_DATA_NESTED = {
-    "priceChanges": [
-        {"changeTime": "2026-01-15T00:00:00", "priceData": {"price": 9000000}},
-    ]
-}
-_STATE_EMPTY: dict[str, object] = {}
-
-
-def test_parity_cian_detail_price_changes() -> None:
-    """`scrape_cian_detail` (admin.py) → fetch_detail → _extract_price_changes."""
-    assert_parity(
-        legacy_fn=legacy_extract_price_changes,
-        kit_fn=kit_extract_price_changes,
-        fixtures=[
-            (_OFFER_FLAT, {}, _STATE_EMPTY),
-            ({}, _OFFER_DATA_NESTED, _STATE_EMPTY),
-            ({}, {}, _STATE_EMPTY),
-        ],
-    )

tradein-mvp/backend/tests/scrapers/test_admin_yandex_kit_parity.py

@@ -1,99 +0,0 @@
-"""Parity-proof: admin.py Yandex debug-роуты (#2305, Group A) — legacy vs scraper_kit.
-
-`app/api/v1/admin.py::scrape_yandex_detail` / `scrape_yandex_valuation` теперь
-зовут `scraper_kit.providers.yandex.{detail,valuation}` вместо
-`app.services.scrapers.yandex_{detail,valuation}` (тот же `/scrape` root-роут —
-`scraper_kit.providers.yandex.serp.YandexRealtyScraper`). Каждая функция ниже —
-чистое, offline-тестируемое parsing-ядро (см. `tests/support/README.md`:
-"Live network/DB в parity-тестах избегайте").
-
-- `_entity_to_lot` — `/scrape` root-роут → `YandexRealtyScraper.fetch_around`
-  конвертирует gate-API entity → ScrapedLot этой функцией (также доказывает
-  ScrapedLot-контракт, общий с `save_listings`/`scripts/ingest_domclick_jsonl.py`).
-
-Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
-финальный шаг scraper_kit-миграции, 0 runtime importers) — `_parse_card_fields` /
-`_parse_house_meta` parity-проверки убраны отсюда вместе с ними; kit-side coverage
-для обеих функций сохранена standalone в `tests/test_yandex_detail_structural.py` /
-`tests/test_yandex_valuation.py`. `yandex_realty` остаётся легаси (не мигрирован) —
-его `_entity_to_lot` parity-проверка не тронута.
-"""
-
-from __future__ import annotations
-
-import os
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from scraper_kit.providers.yandex.serp import _entity_to_lot as kit_entity_to_lot
-
-from app.services.scrapers.yandex_realty import _entity_to_lot as legacy_entity_to_lot
-from tests.support.parity import assert_parity
-
-# gate-API entity (realty.yandex.ru), без creationDate/author — исключает
-# time-dependent поля (days_on_market) из фикстуры (deterministic offline).
-_YANDEX_ENTITY = {
-    "offerId": "12345",
-    "url": "//realty.yandex.ru/offer/12345",
-    "price": {"value": 8500000, "valuePerPart": 157000},
-    "area": {"value": 54.3},
-    "livingSpace": {"value": 32.1},
-    "kitchenSpace": {"value": 9.5},
-    "roomsTotal": 2,
-    "floorsOffered": [5],
-    "floorsTotal": 9,
-    "ceilingHeight": 2.7,
-    "building": {"builtYear": 2015, "buildingType": "MONOLIT"},
-    "location": {
-        "point": {"latitude": 56.83, "longitude": 60.6},
-        "geocoderAddress": "Екатеринбург, ул. Тестовая, 1",
-    },
-    "mainImages": ["//avatars.mds.yandex.net/img1.jpg"],
-}
-
-
-def test_yandex_entity_to_lot_are_class_distinct() -> None:
-    """Sanity: legacy/kit ScrapedLot — разные классы (pydantic), обычный `==` даёт False."""
-    legacy_result = legacy_entity_to_lot(_YANDEX_ENTITY)
-    kit_result = kit_entity_to_lot(_YANDEX_ENTITY)
-
-    assert legacy_result is not None
-    assert kit_result is not None
-    assert type(legacy_result) is not type(kit_result)
-    assert legacy_result.model_dump() == kit_result.model_dump()
-
-
-def test_parity_yandex_serp_entity_to_lot() -> None:
-    """`/scrape` root-роут (admin.py) → YandexRealtyScraper.fetch_around → _entity_to_lot.
-
-    Также доказывает ScrapedLot-контракт, общий с `save_listings`
-    (используется и `/scrape`, и `scripts/ingest_domclick_jsonl.py`).
-    """
-    assert_parity(
-        legacy_fn=legacy_entity_to_lot,
-        kit_fn=kit_entity_to_lot,
-        fixtures=[(_YANDEX_ENTITY,)],
-    )
-
-
-def test_parity_harness_catches_yandex_entity_divergence() -> None:
-    """Harness реально ЛОВИТ расхождение — мутируем price_rub в kit-результате."""
-    import pytest
-
-    from tests.support.parity import ParityMismatchError
-
-    legacy_result = legacy_entity_to_lot(_YANDEX_ENTITY)
-    kit_result = kit_entity_to_lot(_YANDEX_ENTITY)
-    assert legacy_result is not None
-    assert kit_result is not None
-
-    # ScrapedLot — pydantic BaseModel (НЕ dataclass) → model_copy, не dataclasses.replace.
-    mutated_kit_result = kit_result.model_copy(update={"price_rub": kit_result.price_rub + 1})
-
-    with pytest.raises(ParityMismatchError) as exc_info:
-        assert_parity(
-            legacy_fn=lambda: legacy_result,
-            kit_fn=lambda: mutated_kit_result,
-            fixtures=[()],
-        )
-    assert "price_rub" in str(exc_info.value)

tradein-mvp/backend/tests/scrapers/test_avito_unix_date_tz_consistency.py

@@ -3,7 +3,7 @@
 
 До фикса было 3 независимые копии с разными timezone:
   - avito.py (inline в `_build_sort_timestamp_map`) — MSK (документировано #726)
-  - avito_houses.py — UTC
+  - avito_houses.py (легаси, удалён #2397 Part E2) — UTC
   - avito_imv.py (легаси, удалён #2277) — naive/system-local
     (`datetime.fromtimestamp(ts)` без tz)
 
@@ -17,6 +17,11 @@ avito_imv_getdata.json: removedDate=1756077885 → UTC 2025-08-24, MSK 2025-08-2
 kit-эквивалент `scraper_kit.providers.avito.imv`, который использует ТОТ ЖЕ паттерн
 (`from scraper_kit.providers.avito.shared import _unix_to_date`) — регрессия
 "3 независимые копии с разными TZ" остаётся под наблюдением и на kit-стороне.
+
+Легаси `app.services.scrapers.avito_houses` удалён (#2397 Part E2, 0 runtime
+importers) — houses-ветка consolidation-проверки переведена на kit-эквивалент
+`scraper_kit.providers.avito.houses`, который использует ТОТ ЖЕ паттерн
+(`from scraper_kit.providers.avito.shared import RUS_MONTHS, _unix_to_date`).
 """
 
 from __future__ import annotations
@@ -26,10 +31,11 @@ from datetime import date
 
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
 
+from scraper_kit.providers.avito import houses as kit_avito_houses
 from scraper_kit.providers.avito import imv as kit_avito_imv
 from scraper_kit.providers.avito import shared as kit_avito_shared
 
-from app.services.scrapers import avito, avito_houses, avito_shared
+from app.services.scrapers import avito, avito_shared
 
 # Живой пример из fixtures/avito_imv_getdata.json (placementHistory.items[0].removedDate) —
 # именно тот timestamp, на котором UTC и MSK расходились в date() ДО фикса
@@ -44,14 +50,15 @@ def test_shared_unix_to_date_uses_msk() -> None:
 
 
 def test_avito_houses_reexports_shared_unix_to_date() -> None:
-    """avito_houses.py больше не держит свою копию — импортирует общую функцию."""
+    """kit houses.py не держит свою копию — импортирует общую функцию (тот же паттерн,
-    assert avito_houses._unix_to_date is avito_shared._unix_to_date
+    что и легаси avito_houses.py до удаления, #2397 Part E2)."""
-    assert avito_houses._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
+    assert kit_avito_houses._unix_to_date is kit_avito_shared._unix_to_date
+    assert kit_avito_houses._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
 
 
 def test_kit_avito_imv_reexports_shared_unix_to_date() -> None:
     """kit imv.py не держит свою копию — импортирует общую функцию (тот же паттерн,
-    что и легаси avito_houses.py/avito_shared.py после фикса #2129)."""
+    что и kit houses.py/avito_shared.py после фикса #2129)."""
     assert kit_avito_imv._unix_to_date is kit_avito_shared._unix_to_date
     assert kit_avito_imv._unix_to_date(_EPOCH_SECONDS) == _EXPECTED_MSK_DATE
 
@@ -66,11 +73,11 @@ def test_avito_serp_sort_timestamp_matches_shared_date() -> None:
     )
     ts_map = avito._build_sort_timestamp_map(html)
     assert ts_map["8043936560"] == _EXPECTED_MSK_DATE
-    # Легаси avito/avito_houses/avito_shared + kit avito_imv все согласованы на одном
+    # Легаси avito/avito_shared + kit avito_houses/avito_imv все согласованы на одном
     # моменте времени:
     assert (
         ts_map["8043936560"]
-        == avito_houses._unix_to_date(_EPOCH_SECONDS)
+        == kit_avito_houses._unix_to_date(_EPOCH_SECONDS)
         == kit_avito_imv._unix_to_date(_EPOCH_SECONDS)
         == avito_shared._unix_to_date(_EPOCH_SECONDS)
     )
@@ -81,7 +88,7 @@ def test_rus_months_single_source() -> None:
     from app.services.scrapers import avito_detail
 
     assert avito_detail.RUS_MONTHS is avito_shared.RUS_MONTHS
-    assert avito_houses.RUS_MONTHS is avito_shared.RUS_MONTHS
+    assert kit_avito_houses.RUS_MONTHS is kit_avito_shared.RUS_MONTHS
     assert avito_shared.RUS_MONTHS["декабря"] == 12
 
 

tradein-mvp/backend/tests/scrapers/test_cian_exhaustive.py

@@ -1,792 +0,0 @@
-"""Тесты exhaustive-загрузки Cian: fetch_all_secondary + _walk_price_range.
-
-Мокаем сетевые вызовы (_fetch_page_html, _extract_total_offers, sleep_between_requests)
-для детерминированного контроля поведения без реального HTTP.
-"""
-
-from __future__ import annotations
-
-import os
-from unittest.mock import AsyncMock, MagicMock, patch
-
-# Settings requires DATABASE_URL at import time — set dummy DSN before any app import.
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
-
-import logging
-
-import pytest
-
-from app.services.scrapers.base import ScrapedLot
-from app.services.scrapers.cian import (
-    _CIAN_OFFERS_PER_PAGE,
-    _MAX_PRICE,
-    _MIN_BRACKET,
-    CianScraper,
-)
-from app.services.scrapers.price_brackets import get_price_seed_brackets
-
-_SEED = get_price_seed_brackets()
-
-
-def _make_lot(source_id: str, price: int = 3_000_000) -> ScrapedLot:
-    """Минимальный ScrapedLot для теста."""
-    return ScrapedLot(
-        source="cian",
-        source_url=f"https://ekb.cian.ru/sale/flat/{source_id}/",
-        source_id=source_id,
-        address="Екатеринбург",
-        price_rub=price,
-    )
-
-
-def _make_html_stub(total_offers: int, lot_ids: list[str]) -> str:
-    """Заглушка HTML — не парсится напрямую; используется через мокнутые методы."""
-    return f"<html>total={total_offers} ids={lot_ids}</html>"
-
-
-@pytest.fixture
-def scraper() -> CianScraper:
-    """CianScraper без реального __aenter__ (без HTTP warm-up)."""
-    s = CianScraper.__new__(CianScraper)
-    s.name = "cian"
-    s.base_url = "https://ekb.cian.ru"
-    s.request_delay_sec = 0.0
-    s._browser = MagicMock()  # не None → assert self._browser is not None пройдёт
-    return s
-
-
-@pytest.mark.asyncio
-async def test_fetch_all_secondary_splits_on_cap(scraper: CianScraper) -> None:
-    """totalOffers > cap → бакет разбивается; в итоге оба суб-бакета пагинируются."""
-    # Настройка: rooms=(1,), один бакет [0, None]
-    # Probe [0, None]: totalOffers=3000 > cap=1400 → split на [0, MAX_PRICE]
-    # Probe [0, MAX_PRICE]: totalOffers=3000 > 1400 → split на [0, mid] и [mid+1, MAX_PRICE]
-    # Probe [0, mid] (mid = MAX_PRICE//2 = 100_000_000): totalOffers=800 <= cap → пагинировать
-    # Probe [mid+1, MAX_PRICE]: totalOffers=700 <= cap → пагинировать
-
-    mid = _MAX_PRICE // 2
-
-    call_log: list[tuple] = []
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        call_log.append((rooms, page, min_price, max_price))
-        return f"<html>page={page} min={min_price} max={max_price}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        # Парсим min/max из fake html
-        import re
-
-        min_m = re.search(r"min=(\w+)", html)
-        max_m = re.search(r"max=(\w+)", html)
-        min_p = int(min_m.group(1)) if min_m and min_m.group(1) != "None" else 0
-        max_p = int(max_m.group(1)) if max_m and max_m.group(1) != "None" else _MAX_PRICE
-
-        # Имитируем разделение: весь диапазон > 1400, левая и правая половины <= 1400
-        if max_p is None or max_p >= _MAX_PRICE:
-            if min_p == 0:
-                return 3000  # весь диапазон — нужно split
-        if max_p == mid:
-            return 800  # левая половина — пагинировать
-        if min_p == mid + 1:
-            return 700  # правая половина — пагинировать
-        return 3000  # всё остальное — split
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        import re
-
-        page_m = re.search(r"page=(\d+)", html)
-        min_m = re.search(r"min=(\w+)", html)
-        max_m = re.search(r"max=(\w+)", html)
-        page = int(page_m.group(1)) if page_m else 1
-        min_p = min_m.group(1) if min_m else "0"
-        max_p = max_m.group(1) if max_m else "MAX"
-        # Возвращаем по 2 лота на первой странице каждого пагинируемого бакета
-        if page == 1:
-            return [
-                _make_lot(f"lot_{min_p}_{max_p}_p{page}_1"),
-                _make_lot(f"lot_{min_p}_{max_p}_p{page}_2"),
-            ]
-        return []  # страница 2+ — пусто → early stop
-
-    async def fake_sleep(*_: object, **__: object) -> None:
-        pass
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        lots = await scraper.fetch_all_secondary(
-            rooms_buckets=[(1,)],
-            price_cap_per_bucket=1400,
-        )
-
-    # Должны получить лоты из ОБОИХ суб-бакетов после split
-    assert len(lots) > 0, "Должны получить лоты после разделения бакета"
-
-    # Проверяем что был probe с большим диапазоном И с sub-бакетами
-    probed_ranges = [(min_p, max_p) for (_, page, min_p, max_p) in call_log if page == 1]
-    # Должен быть probe [0, None] или [0, MAX] — начальный probe всего диапазона
-    initial_probes = [r for r in probed_ranges if r[0] is None or r[0] == 0]
-    assert (
-        len(initial_probes) >= 1
-    ), f"Ожидался probe всего диапазона, probed_ranges={probed_ranges}"
-
-
-@pytest.mark.asyncio
-async def test_fetch_all_secondary_paginates_when_under_cap(scraper: CianScraper) -> None:
-    """totalOffers <= cap → пагинирует бакет полностью (страница за страницей)."""
-    pages_fetched: list[int] = []
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        pages_fetched.append(page)
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return 56  # 56 офферов → ceil(56/28) = 2 страницы
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        import re
-
-        page_m = re.search(r"page=(\d+)", html)
-        page = int(page_m.group(1)) if page_m else 1
-        if page <= 2:
-            return [_make_lot(f"lot_p{page}_{i}") for i in range(28)]
-        return []
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        lots = await scraper.fetch_all_secondary(
-            rooms_buckets=[(2,)],
-            price_cap_per_bucket=1400,
-        )
-
-    # 28 лотов на стр 1 + 28 на стр 2 = 56, дедуп по source_id
-    assert len(lots) == 56, f"Ожидалось 56 уникальных лотов, получено {len(lots)}"
-    # Должно быть 2 страницы пагинации (page=1 probe + page=2)
-    assert 2 in pages_fetched, "Должна быть запрошена страница 2"
-
-
-@pytest.mark.asyncio
-async def test_fetch_all_secondary_min_bracket_guard(scraper: CianScraper) -> None:
-    """Если hi - lo < MIN_BRACKET → пагинируем как есть (не делим бесконечно)."""
-    call_count = 0
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        nonlocal call_count
-        call_count += 1
-        if call_count > 100:
-            raise RuntimeError("Бесконечная рекурсия — MIN_BRACKET guard не сработал")
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        # Всегда > cap → без guard делилось бы бесконечно
-        return 5000
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        return []  # пустой → early stop в пагинации
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        # Запускаем с очень узким диапазоном — guard должен остановить деление
-        await scraper._walk_price_range(
-            rooms=(1,),
-            lo=1_000_000,
-            hi=1_000_000 + _MIN_BRACKET - 1,  # bracket < MIN_BRACKET
-            seen={},
-            price_cap_per_bucket=1400,
-            max_pages_per_bucket=54,
-        )
-
-    # Guard сработал: нет RuntimeError (бесконечная рекурсия не возникла).
-    # Параллельная пагинация запускает до max_pages_per_bucket=54 страниц единовременно
-    # (все страницы бакета параллельны), поэтому call_count может быть до 54.
-    # Важно что нет рекурсии (bracket < MIN_BRACKET → один проход пагинации, не деление).
-    assert call_count <= 55, f"Слишком много вызовов ({call_count}) — guard не сработал"
-
-
-@pytest.mark.asyncio
-async def test_fetch_all_secondary_on_bucket_called_per_leaf(scraper: CianScraper) -> None:
-    """on_bucket вызывается после каждого leaf-бакета с лотами бакета."""
-    # totalOffers=56 ≤ cap → leaf-бакет, пагинируется параллельно 2 страницы
-    bucket_calls: list[tuple[str, list[ScrapedLot]]] = []
-
-    def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
-        bucket_calls.append((bucket_key, list(lots)))
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return 56  # ceil(56/28) = 2 страницы
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        import re
-
-        page_m = re.search(r"page=(\d+)", html)
-        page = int(page_m.group(1)) if page_m else 1
-        return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        await scraper.fetch_all_secondary(
-            rooms_buckets=[(1,)],
-            price_cap_per_bucket=1400,
-            concurrency=3,
-            on_bucket=fake_on_bucket,
-        )
-
-    # on_bucket вызван по разу на каждый seed-брекет (все непустые leaf-бакеты).
-    assert len(bucket_calls) == len(
-        _SEED
-    ), f"Ожидалось {len(_SEED)} вызовов on_bucket, получено {len(bucket_calls)}"
-    key, lots = bucket_calls[0]
-    # Ключ бакета содержит room_label:lo:hi
-    assert "room1" in key, f"bucket_key должен содержать room1, got {key!r}"
-    # Лоты из обеих страниц переданы в on_bucket
-    assert len(lots) == 20, f"Ожидалось 20 лотов в on_bucket (2 стр × 10), получено {len(lots)}"
-
-
-@pytest.mark.asyncio
-async def test_fetch_all_secondary_on_bucket_cancel_stops_run(scraper: CianScraper) -> None:
-    """Если on_bucket кидает RuntimeError('cancelled') — прогон прерывается."""
-
-    def cancel_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
-        raise RuntimeError("cancelled")
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return 28  # один лист → 1 страница
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        return [_make_lot("lot_1")]
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        with pytest.raises(RuntimeError, match="cancelled"):
-            await scraper.fetch_all_secondary(
-                rooms_buckets=[(1,), (2,)],
-                price_cap_per_bucket=1400,
-                on_bucket=cancel_on_bucket,
-            )
-
-
-@pytest.mark.asyncio
-async def test_fetch_all_secondary_concurrent_pages_deduped(scraper: CianScraper) -> None:
-    """Параллельная пагинация не ломает дедупликацию по source_id."""
-    fetch_pages: list[int] = []
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        fetch_pages.append(page)
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return 84  # ceil(84/28) = 3 страницы
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        import re
-
-        page_m = re.search(r"page=(\d+)", html)
-        page = int(page_m.group(1)) if page_m else 1
-        # Страница 1 и 2 дают уникальные лоты; страница 3 дублирует страницу 1
-        if page == 3:
-            return [_make_lot(f"lot_p1_{i}") for i in range(10)]  # дубликаты
-        return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        lots = await scraper.fetch_all_secondary(
-            rooms_buckets=[(1,)],
-            price_cap_per_bucket=1400,
-            concurrency=5,
-        )
-
-    # Страницы 1+2 = 20 уникальных, страница 3 дублирует → итого 20
-    assert len(lots) == 20, f"Ожидалось 20 уникальных лотов (дедуп), получено {len(lots)}"
-    # Все 3 страницы запрошены (параллельно или нет — нам важен факт)
-    assert 3 in fetch_pages or 3 in [
-        p for p in fetch_pages
-    ], f"Страница 3 не была запрошена, fetch_pages={fetch_pages}"
-
-
-@pytest.mark.asyncio
-async def test_skip_buckets_skips_pagination_and_on_bucket(scraper: CianScraper) -> None:
-    """skip_buckets: для скипнутого бакета пагинация не вызывается, on_bucket не вызывается."""
-    fetch_calls: list[tuple] = []
-    on_bucket_calls: list[str] = []
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        fetch_calls.append((rooms, page, min_price, max_price))
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return 28  # один leaf-бакет, 1 страница
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        return [_make_lot("lot_1")]
-
-    def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
-        on_bucket_calls.append(bucket_key)
-
-    # Скипаем ВСЕ seed-бакеты комнатности room1. Закрытые → ':<hi-1>', открытый → ':open'.
-    skip_set: set[str] = set()
-    for br_lo, br_hi in _SEED:
-        hi_repr = "open" if br_hi is None else str(br_hi - 1)
-        skip_set.add(f"room1:{br_lo}:{hi_repr}")
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        lots = await scraper.fetch_all_secondary(
-            rooms_buckets=[(1,)],
-            price_cap_per_bucket=1400,
-            on_bucket=fake_on_bucket,
-            skip_buckets=skip_set,
-        )
-
-    # Probe (page=1) ВСЁ РАВНО вызывается на каждый seed-брекет (нужен для split-решения),
-    # но НЕ должно быть страниц пагинации (все бакеты в skip_buckets).
-    # Probe = page=1; пагинация пропущена ДО fetch остальных страниц → ровно len(_SEED) probe.
-    assert len(fetch_calls) == len(
-        _SEED
-    ), f"Ожидалось {len(_SEED)} probe-вызовов, получено {len(fetch_calls)}: {fetch_calls}"
-    assert all(c[1] == 1 for c in fetch_calls), "Все вызовы должны быть probe page=1"
-    # on_bucket не вызывается для skip-бакетов
-    assert (
-        len(on_bucket_calls) == 0
-    ), f"on_bucket не должен вызываться для skip-бакетов, got: {on_bucket_calls}"
-    # lots пустой — ничего не собрали (пагинация пропущена)
-    assert len(lots) == 0, f"Ожидалось 0 лотов для skip-бакетов, got {len(lots)}"
-
-
-@pytest.mark.asyncio
-async def test_on_bucket_receives_key_and_lots(scraper: CianScraper) -> None:
-    """on_bucket получает (bucket_key, lots) — bucket_key в формате room_label:lo:hi."""
-    received: list[tuple[str, int]] = []  # (key, len(lots))
-
-    def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
-        received.append((bucket_key, len(lots)))
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return 28  # 1 страница
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        import re
-
-        page_m = re.search(r"page=(\d+)", html)
-        page = int(page_m.group(1)) if page_m else 1
-        return [_make_lot(f"lot_p{page}_{i}") for i in range(5)]
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        await scraper.fetch_all_secondary(
-            rooms_buckets=[(2,)],
-            price_cap_per_bucket=1400,
-            on_bucket=fake_on_bucket,
-        )
-
-    assert len(received) == len(
-        _SEED
-    ), f"Ожидалось {len(_SEED)} вызовов on_bucket, получено {len(received)}"
-    keys = [k for (k, _c) in received]
-    # Закрытые брекеты: hi = br_hi - 1. Открытый (последний): ':open'.
-    first_lo, first_hi = _SEED[0]
-    assert first_hi is not None
-    assert keys[0] == f"room2:{first_lo}:{first_hi - 1}", f"Неверный ключ: {keys[0]!r}"
-    last_lo, last_hi = _SEED[-1]
-    assert last_hi is None
-    assert keys[-1] == f"room2:{last_lo}:open", f"Открытый ключ неверен: {keys[-1]!r}"
-    assert all(count == 5 for (_k, count) in received)
-
-
-# ── secondary_only фильтр ─────────────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_secondary_only_drops_novostroyki(scraper: CianScraper) -> None:
-    """secondary_only=True: новостройки (listing_segment=='novostroyki') не попадают
-    в on_bucket и в итоговый список; вторичка и None-сегмент — пропускаются.
-    """
-    on_bucket_calls: list[list[ScrapedLot]] = []
-
-    def fake_on_bucket(bucket_key: str, lots: list[ScrapedLot]) -> None:
-        on_bucket_calls.append(list(lots))
-
-    # Микс лотов: 2 новостройки + 2 вторички + 1 None-сегмент
-    mixed_lots = [
-        ScrapedLot(
-            source="cian",
-            source_url="https://ekb.cian.ru/sale/flat/nb_1/",
-            source_id="nb_1",
-            address="ЖК Новый",
-            price_rub=5_000_000,
-            listing_segment="novostroyki",
-        ),
-        ScrapedLot(
-            source="cian",
-            source_url="https://ekb.cian.ru/sale/flat/nb_2/",
-            source_id="nb_2",
-            address="ЖК Новый 2",
-            price_rub=6_000_000,
-            listing_segment="novostroyki",
-        ),
-        ScrapedLot(
-            source="cian",
-            source_url="https://ekb.cian.ru/sale/flat/vt_1/",
-            source_id="vt_1",
-            address="ул. Ленина",
-            price_rub=3_500_000,
-            listing_segment="vtorichka",
-        ),
-        ScrapedLot(
-            source="cian",
-            source_url="https://ekb.cian.ru/sale/flat/vt_2/",
-            source_id="vt_2",
-            address="ул. Мира",
-            price_rub=4_000_000,
-            listing_segment="vtorichka",
-        ),
-        ScrapedLot(
-            source="cian",
-            source_url="https://ekb.cian.ru/sale/flat/none_seg/",
-            source_id="none_seg",
-            address="ул. Пушкина",
-            price_rub=3_000_000,
-            listing_segment=None,
-        ),
-    ]
-
-    page_call_count = 0
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        nonlocal page_call_count
-        page_call_count += 1
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return 5  # <= cap → leaf-бакет
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        # Первая страница отдаёт весь микс; вторая — пусто
-        import re
-
-        page_m = re.search(r"page=(\d+)", html)
-        page = int(page_m.group(1)) if page_m else 1
-        return mixed_lots if page == 1 else []
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        lots = await scraper.fetch_all_secondary(
-            rooms_buckets=[(1,)],
-            price_cap_per_bucket=1400,
-            secondary_only=True,
-            on_bucket=fake_on_bucket,
-        )
-
-    # Итоговый список: только vtorichka + None-сегмент (3 лота), без новостроек
-    assert len(lots) == 3, f"Ожидалось 3 лота (без novostroyki), получено {len(lots)}"
-    lot_ids = {lot.source_id for lot in lots}
-    assert "nb_1" not in lot_ids, "nb_1 (novostroyki) не должен присутствовать"
-    assert "nb_2" not in lot_ids, "nb_2 (novostroyki) не должен присутствовать"
-    assert "vt_1" in lot_ids, "vt_1 (vtorichka) должен присутствовать"
-    assert "vt_2" in lot_ids, "vt_2 (vtorichka) должен присутствовать"
-    assert "none_seg" in lot_ids, "none_seg (listing_segment=None) должен присутствовать"
-
-    # on_bucket вызывается на каждый seed-брекет (тот же микс на page=1 каждого),
-    # но новостройки отфильтрованы в каждом вызове.
-    assert len(on_bucket_calls) == len(
-        _SEED
-    ), f"Ожидалось {len(_SEED)} вызовов on_bucket, получено {len(on_bucket_calls)}"
-    for call_lots in on_bucket_calls:
-        bucket_ids = {lot.source_id for lot in call_lots}
-        assert "nb_1" not in bucket_ids
-        assert "nb_2" not in bucket_ids
-        assert len(call_lots) == 3
-
-
-@pytest.mark.asyncio
-async def test_secondary_only_false_passes_all(scraper: CianScraper) -> None:
-    """secondary_only=False: новостройки НЕ отфильтровываются — все лоты проходят."""
-    mixed_lots = [
-        ScrapedLot(
-            source="cian",
-            source_url="https://ekb.cian.ru/sale/flat/nb_1/",
-            source_id="nb_1",
-            address="ЖК Новый",
-            price_rub=5_000_000,
-            listing_segment="novostroyki",
-        ),
-        ScrapedLot(
-            source="cian",
-            source_url="https://ekb.cian.ru/sale/flat/vt_1/",
-            source_id="vt_1",
-            address="ул. Ленина",
-            price_rub=3_500_000,
-            listing_segment="vtorichka",
-        ),
-    ]
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return 2
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        import re
-
-        page_m = re.search(r"page=(\d+)", html)
-        page = int(page_m.group(1)) if page_m else 1
-        return mixed_lots if page == 1 else []
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        lots = await scraper.fetch_all_secondary(
-            rooms_buckets=[(1,)],
-            price_cap_per_bucket=1400,
-            secondary_only=False,
-        )
-
-    assert len(lots) == 2, f"Ожидалось 2 лота (фильтр выключен), получено {len(lots)}"
-    lot_ids = {lot.source_id for lot in lots}
-    assert "nb_1" in lot_ids, "nb_1 должен присутствовать при secondary_only=False"
-    assert "vt_1" in lot_ids
-
-
-# ── #1393: tail-loss warning при превышении page cap ─────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_walk_price_range_warns_on_page_cap_tail_loss(
-    scraper: CianScraper, caplog: pytest.LogCaptureFixture
-) -> None:
-    """#1393: когда totalOffers > max_pages_per_bucket × offers_per_page,
-    _walk_price_range логирует WARNING с числом потерянных офферов.
-
-    Сценарий: bracket < _MIN_BRACKET (guard срабатывает → force-paginate),
-    totalOffers = max_pages * offers_per_page + 50 → tail_loss = 50.
-    """
-    max_pages_cap = 5  # нарочно мало, чтобы быстро дойти до cap
-    offers_per_page = _CIAN_OFFERS_PER_PAGE  # 28
-    # total > cap (2000 > любой price_cap_per_bucket=1400) И bracket < _MIN_BRACKET
-    # → force-paginate path; pages_needed = ceil(2000/28) = 72 > max_pages_cap=5
-    total_offers = max_pages_cap * offers_per_page + 50  # 190; tail_loss=50
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return total_offers
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        import re
-
-        page_m = re.search(r"page=(\d+)", html)
-        page = int(page_m.group(1)) if page_m else 1
-        return [_make_lot(f"lot_p{page}_{i}") for i in range(offers_per_page)]
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-        caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"),
-    ):
-        await scraper._walk_price_range(
-            rooms=(1,),
-            # Узкий диапазон: bracket < _MIN_BRACKET → force-paginate guard
-            lo=1_000_000,
-            hi=1_000_000 + _MIN_BRACKET - 1,
-            seen={},
-            price_cap_per_bucket=1400,  # total(190) < 1400 → нет split, только leaf
-            max_pages_per_bucket=max_pages_cap,
-        )
-
-    # Должно быть предупреждение о tail-loss
-    tail_warnings = [
-        r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING
-    ]
-    assert tail_warnings, (
-        f"Ожидалось WARNING о tail loss, но записей нет. "
-        f"Все warnings: {[r.message for r in caplog.records if r.levelno == logging.WARNING]}"
-    )
-    # Сообщение должно содержать число потерянных офферов (50)
-    assert any(
-        "50" in r.message for r in tail_warnings
-    ), f"WARNING должен содержать tail_loss=50, got: {[r.message for r in tail_warnings]}"
-
-
-@pytest.mark.asyncio
-async def test_walk_price_range_no_warning_when_within_page_cap(
-    scraper: CianScraper, caplog: pytest.LogCaptureFixture
-) -> None:
-    """#1393: когда totalOffers <= max_pages_per_bucket × offers_per_page,
-    tail-loss WARNING не должен эмитироваться (данные собраны полностью).
-    """
-    max_pages_cap = 54
-    total_offers = 56  # ceil(56/28) = 2 страницы < 54 cap → нет потерь
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return total_offers
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        import re
-
-        page_m = re.search(r"page=(\d+)", html)
-        page = int(page_m.group(1)) if page_m else 1
-        return [_make_lot(f"lot_p{page}_{i}") for i in range(10)]
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-        caplog.at_level(logging.WARNING, logger="app.services.scrapers.cian"),
-    ):
-        await scraper._walk_price_range(
-            rooms=(1,),
-            lo=0,
-            hi=_MAX_PRICE,
-            seen={},
-            price_cap_per_bucket=1400,
-            max_pages_per_bucket=max_pages_cap,
-        )
-
-    tail_warnings = [
-        r for r in caplog.records if "tail loss" in r.message and r.levelno == logging.WARNING
-    ]
-    assert not tail_warnings, (
-        f"Не ожидалось WARNING о tail loss при total={total_offers} <= page cap, "
-        f"но получено: {[r.message for r in tail_warnings]}"
-    )
-
-
-@pytest.mark.asyncio
-async def test_walk_price_range_open_bracket_no_split(scraper: CianScraper) -> None:
-    """Открытый брекет (hi=None) не делится даже при total > cap — leaf без maxprice.
-
-    bucket_key оканчивается на ':open'; maxprice в запросах всегда None (без потолка).
-    """
-    on_bucket_keys: list[str] = []
-    seen_max: list[int | None] = []
-
-    async def fake_fetch_page_html(
-        rooms: tuple, page: int, min_price: int | None, max_price: int | None
-    ) -> str:
-        seen_max.append(max_price)
-        return f"<html>page={page}</html>"
-
-    def fake_extract_total_offers(html: str) -> int | None:
-        return 5000  # > cap, но открытый брекет делить нельзя → leaf
-
-    def fake_parse_serp_html(html: str) -> list[ScrapedLot]:
-        import re
-
-        page_m = re.search(r"page=(\d+)", html)
-        page = int(page_m.group(1)) if page_m else 1
-        return [_make_lot(f"lot_p{page}")] if page == 1 else []
-
-    with (
-        patch.object(scraper, "_fetch_page_html", side_effect=fake_fetch_page_html),
-        patch.object(scraper, "_extract_total_offers", side_effect=fake_extract_total_offers),
-        patch.object(scraper, "_parse_serp_html", side_effect=fake_parse_serp_html),
-        patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock),
-        patch.object(scraper, "_rotate_ip", return_value=False),
-    ):
-        await scraper._walk_price_range(
-            rooms=(1,),
-            lo=250_000_000,
-            hi=None,
-            seen={},
-            price_cap_per_bucket=1400,
-            max_pages_per_bucket=54,
-            on_bucket=lambda k, lots: on_bucket_keys.append(k),
-        )
-
-    # Открытый брекет: ровно один leaf-бакет с ключом ':open', maxprice всегда None.
-    assert on_bucket_keys == [
-        "room1:250000000:open"
-    ], f"Открытый брекет должен дать один ':open'-бакет, got {on_bucket_keys}"
-    assert all(
-        m is None for m in seen_max
-    ), f"Открытый брекет: maxprice всегда None (без потолка), got {seen_max}"

tradein-mvp/backend/tests/scrapers/test_domclick.py

@@ -1,630 +0,0 @@
-"""Unit-тесты для DomClick BFF scraper (#1968).
-
-Тестируем:
-- _extract_json: bare + <pre>-wrapped → dict; QRATOR-маркер → DomClickBlockedError
-- _build_offers_url / _build_count_url: корректные параметры, "5+" → "5%2B",
-  offers имеет offset/limit, count — нет
-- _map_item: маппинг realistic offer dict → ScrapedLot; price=0 → None; studio
-- _is_geo_ok: гео-гард по bbox + offerRegionName
-- _sweep_bucket: price-split при count > OFFSET_CAP
-- DomClickScraper: source/name, fetch_around raises NotImplementedError
-"""
-
-from __future__ import annotations
-
-import asyncio
-from typing import Any
-
-import pytest
-
-from app.services.scrapers.domclick import (
-    LTE_MAX,
-    OFFSET_CAP,
-    DomClickScraper,
-    _build_count_url,
-    _build_offers_url,
-    _extract_json,
-)
-from app.services.scrapers.domclick_exceptions import DomClickBlockedError
-
-# ── _extract_json ─────────────────────────────────────────────────────────────
-
-
-def test_extract_json_bare_body() -> None:
-    """Bare JSON body (без HTML-обёртки) корректно парсится."""
-    html = '{"result": {"items": [], "snippetsCount": 42}}'
-    data = _extract_json(html)
-    assert data["result"]["snippetsCount"] == 42
-
-
-def test_extract_json_pre_wrapped() -> None:
-    """JSON в <pre>-теге корректно парсится."""
-    html = (
-        "<html><head><title>DomClick</title></head>"
-        '<body><pre>{"result": {"items": [{"id": 1}]}}</pre></body></html>'
-    )
-    data = _extract_json(html)
-    assert data["result"]["items"][0]["id"] == 1
-
-
-def test_extract_json_qrator_marker_raises() -> None:
-    """HTML с QRATOR-маркером вызывает DomClickBlockedError."""
-    blocked_html = (
-        "<html><head><title>QRATOR protection</title></head>"
-        "<body>Access denied by bot_mitigation system</body></html>"
-    )
-    with pytest.raises(DomClickBlockedError):
-        _extract_json(blocked_html)
-
-
-def test_extract_json_captcha_raises() -> None:
-    """HTML с captcha-маркером вызывает DomClickBlockedError."""
-    html = "<html><body>Please solve the captcha to continue</body></html>"
-    with pytest.raises(DomClickBlockedError):
-        _extract_json(html)
-
-
-def test_extract_json_no_json_raises() -> None:
-    """HTML без JSON-объекта вызывает ValueError."""
-    with pytest.raises(ValueError, match="No JSON"):
-        _extract_json("<html><body>plain text</body></html>")
-
-
-# ── _build_offers_url ────────────────────────────────────────────────────────
-
-
-def test_build_offers_url_basic_params() -> None:
-    url = _build_offers_url("1", None, None, 0)
-    assert "deal_type=sale" in url
-    assert "category=living" in url
-    assert "offer_type=flat" in url
-    assert "rooms=1" in url
-    assert "offset=0" in url
-    assert "limit=20" in url
-    assert "sale_price__gte" not in url
-    assert "sale_price__lte" not in url
-
-
-def test_build_offers_url_5plus_encoded() -> None:
-    """rooms='5+' должен кодироваться как '5%2B' в URL."""
-    url = _build_offers_url("5+", None, None, 0)
-    assert "5%2B" in url
-    # literal '+' в URL не должно быть для значения rooms
-    assert "rooms=5+" not in url
-
-
-def test_build_offers_url_with_price_range() -> None:
-    url = _build_offers_url("2", 3_000_000, 6_000_000, 40)
-    assert "sale_price__gte=3000000" in url
-    assert "sale_price__lte=6000000" in url
-    assert "offset=40" in url
-
-
-def test_build_offers_url_ekb_guid_present() -> None:
-    url = _build_offers_url("3", None, None, 0)
-    assert "0d475b79-88de-4054-818c-37d8f9d0d440" in url
-    assert "aids=20561" in url
-
-
-def test_build_offers_url_bff_host() -> None:
-    url = _build_offers_url("1", None, None, 0)
-    assert url.startswith("https://bff-search-web.domclick.ru/api/offers/v1?")
-
-
-# ── _build_count_url ─────────────────────────────────────────────────────────
-
-
-def test_build_count_url_no_offset_limit() -> None:
-    """count url не должен содержать offset и limit."""
-    url = _build_count_url("2", None, None)
-    assert "offset" not in url
-    assert "limit" not in url
-
-
-def test_build_count_url_5plus_encoded() -> None:
-    url = _build_count_url("5+", None, None)
-    assert "5%2B" in url
-    assert "rooms=5+" not in url
-
-
-def test_build_count_url_with_price() -> None:
-    url = _build_count_url("1", 1_000_000, 4_000_000)
-    assert "sale_price__gte=1000000" in url
-    assert "sale_price__lte=4000000" in url
-
-
-def test_build_count_url_bff_host() -> None:
-    url = _build_count_url("1", None, None)
-    assert url.startswith("https://bff-search-web.domclick.ru/api/offers/count/v1?")
-
-
-# ── _map_item → ScrapedLot ────────────────────────────────────────────────────
-
-_REALISTIC_ITEM: dict[str, Any] = {
-    "id": 2075671636,
-    "path": "https://domclick.ru/card/sale__flat__2075671636",
-    "location": {"lat": 56.838, "lon": 60.612},
-    "address": {"displayName": "Екатеринбург, улица Малышева, 1"},
-    "objectInfo": {"area": 53.5, "rooms": 2, "floor": 5, "isApartment": False},
-    "house": {"floors": 15, "buildYear": 2003},
-    "price": 5_200_000,
-    "squarePrice": 97196,
-    "flatComplex": {"id": 12345, "name": "ЖК Тест", "slug": "zhk-test"},
-    "isRosreestrApproved": True,
-    "publishedDate": "2026-06-01T10:00:00+05:00",
-    "updatedDate": "2026-06-15T12:00:00+05:00",
-    "lastPriceHistoryState": {"direction": "DECREASED"},
-    "offerRegionName": "Екатеринбург",
-}
-
-
-def test_map_item_basic_fields() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    lot = scraper._map_item(_REALISTIC_ITEM)
-    assert lot is not None
-    assert lot.source == "domklik"
-    assert lot.source_id == "2075671636"
-    assert lot.source_url == "https://domclick.ru/card/sale__flat__2075671636"
-    assert lot.lat == pytest.approx(56.838)
-    assert lot.lon == pytest.approx(60.612)
-    assert lot.address == "Екатеринбург, улица Малышева, 1"
-    assert lot.rooms == 2
-    assert lot.area_m2 == pytest.approx(53.5)
-    assert lot.floor == 5
-    assert lot.total_floors == 15
-    assert lot.year_built == 2003
-    assert lot.price_rub == 5_200_000
-    assert lot.price_per_m2 == 97196
-    assert lot.listing_segment == "vtorichka"
-
-
-def test_map_item_publish_date() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    from datetime import date
-
-    lot = scraper._map_item(_REALISTIC_ITEM)
-    assert lot is not None
-    assert lot.publish_date == date(2026, 6, 1)
-
-
-def test_map_item_raw_payload() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    lot = scraper._map_item(_REALISTIC_ITEM)
-    assert lot is not None
-    assert lot.raw_payload is not None
-    assert lot.raw_payload["isRosreestrApproved"] is True
-    assert lot.raw_payload["flatComplex"]["name"] == "ЖК Тест"
-    assert lot.raw_payload["lastPriceHistoryState"]["direction"] == "DECREASED"
-
-
-def test_map_item_studio_force_rooms_zero() -> None:
-    """force_rooms=0 (студия-бакет) перебивает objectInfo.rooms."""
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    item = {**_REALISTIC_ITEM, "objectInfo": {"area": 28.0, "rooms": 1, "floor": 3}}
-    lot = scraper._map_item(item, force_rooms=0)
-    assert lot is not None
-    assert lot.rooms == 0
-
-
-def test_map_item_zero_price_returns_none() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    item = {**_REALISTIC_ITEM, "price": 0}
-    assert scraper._map_item(item) is None
-
-
-def test_map_item_zero_sentinels_become_none() -> None:
-    """buildYear/area/floor/total_floors == 0 (sentinel «неизвестно») → None.
-
-    rooms=0 НЕ затрагивается (валидная студия), но здесь force_rooms не задан и
-    objectInfo.rooms=2, так что rooms остаётся 2.
-    """
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    item = {
-        **_REALISTIC_ITEM,
-        "objectInfo": {"area": 0, "rooms": 2, "floor": 0},
-        "house": {"floors": 0, "buildYear": 0},
-    }
-    lot = scraper._map_item(item)
-    assert lot is not None
-    assert lot.area_m2 is None
-    assert lot.floor is None
-    assert lot.total_floors is None
-    assert lot.year_built is None
-    assert lot.rooms == 2  # rooms 0-sentinel НЕ применяется
-
-
-def test_map_item_rooms_zero_studio_preserved() -> None:
-    """force_rooms=0 даёт rooms=0 (студия) — 0 здесь валиден, не sentinel."""
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    item = {**_REALISTIC_ITEM, "objectInfo": {"area": 24.0, "rooms": 0, "floor": 2}}
-    lot = scraper._map_item(item, force_rooms=0)
-    assert lot is not None
-    assert lot.rooms == 0
-
-
-def test_map_item_price_per_m2_fallback() -> None:
-    """Если squarePrice отсутствует, price_per_m2 вычисляется из price / area_m2."""
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    item = {**_REALISTIC_ITEM, "squarePrice": None}
-    lot = scraper._map_item(item)
-    assert lot is not None
-    assert lot.price_per_m2 == int(5_200_000 / 53.5)
-
-
-# ── _map_item Layer A enrichment (description / repair / minhash / seller) ─────
-
-_VALID_REPAIR_ENUM = {"needs_repair", "standard", "good", "excellent"}
-
-
-def test_map_item_description_repair_minhash() -> None:
-    """Offer с описанием → description set, repair_state выведен, minhash len 32."""
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    item = {
-        **_REALISTIC_ITEM,
-        "description": "Продаётся светлая квартира с евроремонтом, рядом метро.",
-    }
-    lot = scraper._map_item(item)
-    assert lot is not None
-    assert lot.description == "Продаётся светлая квартира с евроремонтом, рядом метро."
-    # «евроремонт» → good (см. repair_state_normalizer._TEXT_PATTERNS)
-    assert lot.repair_state == "good"
-    assert lot.repair_state in _VALID_REPAIR_ENUM
-    assert lot.description_minhash is not None
-    assert len(lot.description_minhash) == 32
-
-
-def test_map_item_description_stripped() -> None:
-    """description триммится; пустой/whitespace → None."""
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    item = {**_REALISTIC_ITEM, "description": "   Без отделки, требуется ремонт.  "}
-    lot = scraper._map_item(item)
-    assert lot is not None
-    assert lot.description == "Без отделки, требуется ремонт."
-    # «без отделк» → needs_repair
-    assert lot.repair_state == "needs_repair"
-
-
-def test_map_item_enrichment_raw_payload() -> None:
-    """Новые SERP-поля попадают в raw_payload (is_sber_collateral и т.д.)."""
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    item = {
-        **_REALISTIC_ITEM,
-        "isSberCollateral": True,
-        "hasDiscount": True,
-        "discountValue": 2.3,
-        "duplicatesOfferCount": 3,
-        "seller": {"company": "Этажи", "agent": "Иван Петров"},
-    }
-    lot = scraper._map_item(item)
-    assert lot is not None
-    assert lot.raw_payload is not None
-    assert lot.raw_payload["is_sber_collateral"] is True
-    assert lot.raw_payload["has_discount"] is True
-    assert lot.raw_payload["discount_value"] == 2.3
-    assert lot.raw_payload["duplicates_offer_count"] == 3
-    assert lot.raw_payload["seller"] == {"company": "Этажи", "agent": "Иван Петров"}
-    # agency_name извлекается из seller.company (приоритет company → agent)
-    assert lot.agency_name == "Этажи"
-
-
-def test_map_item_agency_name_from_agent_dict() -> None:
-    """seller.company отсутствует, agent — dict с name → agency_name = name."""
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    item = {**_REALISTIC_ITEM, "seller": {"agent": {"name": "Мария С."}}}
-    lot = scraper._map_item(item)
-    assert lot is not None
-    assert lot.agency_name == "Мария С."
-    assert lot.raw_payload is not None
-    assert lot.raw_payload["seller"] == {"agent": {"name": "Мария С."}}
-
-
-def test_map_item_no_description_no_crash() -> None:
-    """Offer без description: не падает, description/repair_state/minhash = None."""
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    # _REALISTIC_ITEM не содержит description/seller/enrichment-полей
-    lot = scraper._map_item(_REALISTIC_ITEM)
-    assert lot is not None
-    assert lot.description is None
-    assert lot.repair_state is None
-    assert lot.description_minhash is None
-    assert lot.agency_name is None
-
-
-def test_map_item_enrichment_preserves_existing_raw_payload_and_drops_none() -> None:
-    """Существующие raw_payload-ключи не теряются; None-enrich-ключи отброшены."""
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    lot = scraper._map_item(_REALISTIC_ITEM)
-    assert lot is not None
-    assert lot.raw_payload is not None
-    # Существующие ключи на месте
-    assert lot.raw_payload["isRosreestrApproved"] is True
-    assert lot.raw_payload["flatComplex"]["name"] == "ЖК Тест"
-    assert lot.raw_payload["lastPriceHistoryState"]["direction"] == "DECREASED"
-    assert "updatedDate" in lot.raw_payload
-    # None-значные enrich-ключи отсутствуют (компактный payload)
-    assert "is_sber_collateral" not in lot.raw_payload
-    assert "has_discount" not in lot.raw_payload
-    assert "discount_value" not in lot.raw_payload
-    assert "duplicates_offer_count" not in lot.raw_payload
-    assert "seller" not in lot.raw_payload
-
-
-# ── _is_geo_ok — гео-гард ────────────────────────────────────────────────────
-
-
-def test_is_geo_ok_valid_ekb_item() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    assert scraper._is_geo_ok(_REALISTIC_ITEM) is True
-
-
-def test_is_geo_ok_wrong_region_name() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    item = {**_REALISTIC_ITEM, "offerRegionName": "Москва"}
-    assert scraper._is_geo_ok(item) is False
-
-
-def test_is_geo_ok_out_of_bbox_lat() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    item = {**_REALISTIC_ITEM, "location": {"lat": 55.5, "lon": 60.6}}
-    assert scraper._is_geo_ok(item) is False
-
-
-def test_is_geo_ok_out_of_bbox_lon() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    item = {**_REALISTIC_ITEM, "location": {"lat": 56.8, "lon": 61.5}}
-    assert scraper._is_geo_ok(item) is False
-
-
-def test_is_geo_ok_missing_coords() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    item = {**_REALISTIC_ITEM, "location": {"lat": None, "lon": None}}
-    assert scraper._is_geo_ok(item) is False
-
-
-# ── _sweep_bucket price-split ────────────────────────────────────────────────
-
-
-async def test_sweep_bucket_price_split_boundaries(monkeypatch: pytest.MonkeyPatch) -> None:
-    """_sweep_bucket: top-level unbounded split даёт contiguous non-overlapping ranges.
-
-    rooms="2" count>cap один раз, потом ≤cap → ровно 2 leaf-бакета, покрывающих
-    [0, LTE_MAX] без перекрытия и без зазора: [(0, mid), (mid+1, LTE_MAX)].
-    """
-    call_count: dict[str, int] = {"n": 0}
-    paginate_calls: list[tuple[str, int | None, int | None]] = []
-
-    async def fake_count(
-        self: DomClickScraper,
-        fetcher: object,
-        rooms: str,
-        price_gte: int | None,
-        price_lte: int | None,
-    ) -> int:
-        call_count["n"] += 1
-        # Первый вызов (unbounded) → over cap → price-split; остальные → под cap
-        return OFFSET_CAP + 1 if call_count["n"] == 1 else 1
-
-    async def fake_paginate(
-        self: DomClickScraper,
-        fetcher: object,
-        rooms: str,
-        price_gte: int | None,
-        price_lte: int | None,
-        seen_ids: set[str],
-        out_lots: list,
-        pages: int,
-    ) -> None:
-        paginate_calls.append((rooms, price_gte, price_lte))
-
-    monkeypatch.setattr(DomClickScraper, "_count", fake_count)
-    monkeypatch.setattr(DomClickScraper, "_paginate", fake_paginate)
-
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    scraper.geo_filtered = 0
-    scraper.blocked = False
-    scraper.fetch_errors = 0
-    scraper.request_delay_sec = 0.0
-
-    await scraper._sweep_bucket(
-        fetcher=object(),
-        rooms="2",
-        price_gte=None,
-        price_lte=None,
-        seen_ids=set(),
-        out_lots=[],
-        pages=10,
-    )
-
-    # 1 count call (over cap) + 2 recursive count calls (≤ cap) = 3 total
-    assert call_count["n"] == 3
-    # Both sub-buckets paginated, с точными границами split'а unbounded-диапазона.
-    mid = (0 + LTE_MAX) // 2
-    assert paginate_calls == [("2", 0, mid), ("2", mid + 1, LTE_MAX)]
-    # Sanity: contiguous (нет зазора) + non-overlapping + покрывают весь хвост.
-    assert paginate_calls[0][2] + 1 == paginate_calls[1][1]  # mid + 1
-    assert paginate_calls[0][1] == 0
-    assert paginate_calls[1][2] == LTE_MAX
-
-
-async def test_sweep_bucket_zero_count_no_paginate(monkeypatch: pytest.MonkeyPatch) -> None:
-    """_sweep_bucket при count==0 не вызывает _paginate."""
-    paginate_called = False
-
-    async def fake_count(self: DomClickScraper, *_args: object, **_kwargs: object) -> int:
-        return 0
-
-    async def fake_paginate(self: DomClickScraper, *_args: object, **_kwargs: object) -> None:
-        nonlocal paginate_called
-        paginate_called = True
-
-    monkeypatch.setattr(DomClickScraper, "_count", fake_count)
-    monkeypatch.setattr(DomClickScraper, "_paginate", fake_paginate)
-
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    scraper.geo_filtered = 0
-    scraper.blocked = False
-    scraper.fetch_errors = 0
-    scraper.request_delay_sec = 0.0
-
-    await scraper._sweep_bucket(
-        fetcher=object(),
-        rooms="3",
-        price_gte=None,
-        price_lte=None,
-        seen_ids=set(),
-        out_lots=[],
-        pages=5,
-    )
-    assert paginate_called is False
-
-
-# ── _paginate resilience — mid-bucket parse error ────────────────────────────
-
-
-class _FakeFetcher:
-    """Стаб BrowserFetcher: возвращает заранее заданные ответы по порядку вызовов."""
-
-    def __init__(self, responses: list[str]) -> None:
-        self._responses = responses
-        self._idx = 0
-
-    async def fetch(self, _url: str) -> str:
-        resp = self._responses[min(self._idx, len(self._responses) - 1)]
-        self._idx += 1
-        return resp
-
-
-async def test_paginate_mid_bucket_parse_error_preserves_lots() -> None:
-    """Garbled JSON на page 2 → bucket breaks, page-1 lots survive, fetch_errors++.
-
-    Page 1 — валидный JSON с 20 items (PAGE_SIZE) → продолжаем; page 2 — мусор
-    (ValueError из _extract_json) → fetch_errors++ и break, уже собранные lots целы.
-    """
-    import json
-
-    page1_items = [
-        {
-            "id": 1000 + i,
-            "path": f"https://domclick.ru/card/sale__flat__{1000 + i}",
-            "location": {"lat": 56.84, "lon": 60.6},
-            "address": {"displayName": "Екатеринбург, тест"},
-            "objectInfo": {"area": 40.0, "rooms": 1, "floor": 3},
-            "house": {"floors": 9, "buildYear": 2005},
-            "price": 4_000_000 + i,
-            "squarePrice": 100000,
-            "offerRegionName": "Екатеринбург",
-        }
-        for i in range(20)
-    ]
-    page1 = json.dumps({"result": {"items": page1_items}})
-    garbled = "<html><body><pre>{not valid json at all</pre></body></html>"
-    fetcher = _FakeFetcher([page1, garbled])
-
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    scraper.geo_filtered = 0
-    scraper.blocked = False
-    scraper.fetch_errors = 0
-    scraper.request_delay_sec = 0.0
-
-    out_lots: list = []
-    await scraper._paginate(
-        fetcher=fetcher,
-        rooms="1",
-        price_gte=None,
-        price_lte=None,
-        seen_ids=set(),
-        out_lots=out_lots,
-        pages=10,
-    )
-
-    assert len(out_lots) == 20  # page-1 lots сохранены
-    assert scraper.fetch_errors == 1  # page-2 garbled посчитан
-
-
-async def test_fetch_city_bucket_error_continues_to_next_bucket(
-    monkeypatch: pytest.MonkeyPatch,
-) -> None:
-    """Ошибка в одном бакете НЕ убивает весь sweep — остальные бакеты отрабатывают."""
-    swept: list[str] = []
-
-    async def fake_sweep_bucket(
-        self: DomClickScraper,
-        fetcher: object,
-        rooms: str,
-        price_gte: int | None,
-        price_lte: int | None,
-        seen_ids: set[str],
-        out_lots: list,
-        pages: int,
-    ) -> None:
-        swept.append(rooms)
-        if rooms == "2":
-            raise ValueError("garbled bucket")
-
-    # Заглушка BrowserFetcher через async context manager
-    class _BFStub:
-        async def __aenter__(self) -> _BFStub:
-            return self
-
-        async def __aexit__(self, *_a: object) -> None:
-            return None
-
-        async def fetch(self, _url: str) -> str:
-            return "{}"
-
-    import app.services.scrapers.browser_fetcher as bf_mod
-
-    monkeypatch.setattr(DomClickScraper, "_sweep_bucket", fake_sweep_bucket)
-    monkeypatch.setattr(bf_mod, "BrowserFetcher", lambda **_k: _BFStub())
-
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    scraper.geo_filtered = 0
-    scraper.blocked = False
-    scraper.fetch_errors = 0
-    scraper.request_delay_sec = 0.0
-
-    lots = await scraper.fetch_city(city_id=4, rooms=None, pages=5)
-
-    # Все 6 бакетов посещены несмотря на ошибку в "2"
-    assert swept == ["st", "1", "2", "3", "4", "5+"]
-    assert scraper.fetch_errors == 1
-    assert scraper.blocked is False
-    assert lots == []
-
-
-# ── DomClickScraper — class attrs ─────────────────────────────────────────────
-
-
-def test_scraper_source_name() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    scraper.parse_failures = 0
-    scraper.geo_filtered = 0
-    scraper.blocked = False
-    scraper.request_delay_sec = 0.0
-    assert scraper.name == "domklik"
-    assert scraper.source == "domklik"
-    assert scraper.base_url == "https://domclick.ru"
-
-
-def test_fetch_around_raises() -> None:
-    scraper = DomClickScraper.__new__(DomClickScraper)
-    with pytest.raises(NotImplementedError, match="geo-radius"):
-        asyncio.run(scraper.fetch_around(56.8, 60.6))

tradein-mvp/backend/tests/scrapers/test_domclick_detail.py

@@ -24,9 +24,10 @@ test_cian_detail.py — в проекте нет live-DB фикстуры, ид
 scraper_kit-миграции, 0 runtime importers) — тесты переведены на kit-эквивалент
 `scraper_kit.providers.domclick.detail` без изменения покрытия. Exceptions тоже
 переведены на `scraper_kit.domclick_exceptions` (kit держит свою независимую копию,
-`parse_detail_html`(kit) поднимает именно её — legacy `app.services.scrapers.
+`parse_detail_html`(kit) поднимает именно её). Legacy `app.services.scrapers.
-domclick_exceptions` остаётся живым модулем для `domclick.py`, но здесь больше не
+domclick`/`domclick_exceptions` удалены целиком (#2397 Part E2, 0 runtime
-подходящий класс для `pytest.raises`).
+importers) — `scraper_kit.domclick_exceptions` остаётся единственным живым классом
+для `pytest.raises`.
 """
 
 from __future__ import annotations

tradein-mvp/backend/tests/test_avito_houses_dedup.py

@@ -1,381 +0,0 @@
-"""Tests for avito_houses dedup fix — _persist_house делегирует в match_or_create_house.
-
-Проверяет:
-  1. Два HouseInfo с разными ext_id но одинаковым адресом и координатами
-     → _persist_house возвращает ОДИН house_id (не плодит дубль).
-  2. Enrichment-поля (rating_score, developer_name) реально записываются в UPDATE.
-  3. _persist_house не вызывает db.commit().
-  4. upsert_house и _upsert_house_with_url делегируют в _persist_house.
-  5. #1820: два id-less URL (ext_id=0, без адреса) НЕ создают house_sources(avito,"0").
-
-Использует тот же паттерн mock-DB что в test_matching.py и test_same_building_match.py.
-"""
-
-import json
-import os
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from unittest.mock import MagicMock, patch
-
-from app.services.scrapers.avito_houses import (
-    HouseCatalogEnrichment,
-    HouseInfo,
-    _persist_house,
-    _upsert_house_with_url,
-    save_house_catalog_enrichment,
-    upsert_house,
-)
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def _make_house(ext_id: int = 111, address: str = "ул. Ленина, 5") -> HouseInfo:
-    return HouseInfo(
-        ext_id=ext_id,
-        ext_id_hash="hash_" + str(ext_id),
-        short_address=address,
-        full_address="Свердловская обл., Екатеринбург, " + address,
-        lat=56.8378,
-        lon=60.5946,
-        year_built=2010,
-        house_type="monolith",
-        total_floors=16,
-        passenger_elevators=2,
-        cargo_elevators=1,
-        has_concierge=True,
-        closed_yard=False,
-        material_floors="concrete",
-        hot_water="central",
-        parking_type="underground",
-        has_playground=True,
-        developer_name="ПИК",
-        developer_key="pik",
-        infrastructure_summary="Школа, детсад",
-        infrastructure_walk_distance="5 мин",
-        rating_score=4.5,
-        rating_string="4.5 из 5",
-        reviews_count=120,
-        rating_distribution=[{"score": 5, "count": 80}],
-        map_pins=[{"type": "metro", "name": "Уральская"}],
-        raw_characteristics=[{"title": "Год постройки", "value": "2010"}],
-    )
-
-
-def _make_db_for_persist(canonical_id: int = 42) -> MagicMock:
-    """Mock DB для _persist_house: match_or_create_house ходит через db.execute,
-    поэтому патчим match_or_create_house целиком и проверяем вызов UPDATE.
-    """
-    db = MagicMock()
-    # db.execute для UPDATE houses SET ... — просто возвращаем None (rowcount не важен)
-    db.execute.return_value = MagicMock()
-    return db
-
-
-# ---------------------------------------------------------------------------
-# Test 1: два HouseInfo с разными ext_id, одинаковый адрес → один house_id
-# ---------------------------------------------------------------------------
-
-
-def test_persist_house_same_address_returns_same_id():
-    """Два HouseInfo с разными ext_id но одинаковым адресом → один canonical house_id.
-
-    Моделируем случай: первый вызов (ext_id=111) → match_or_create_house вернул 42 (new).
-    Второй вызов (ext_id=222, тот же адрес) → match_or_create_house вернул 42 (fingerprint).
-    Оба _persist_house должны вернуть 42.
-    """
-    db = MagicMock()
-    db.execute.return_value = MagicMock()
-
-    h1 = _make_house(ext_id=111)
-    h2 = _make_house(ext_id=222)  # другой ext_id, тот же адрес/координаты
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-        side_effect=[(42, 1.0, "new"), (42, 0.9, "fingerprint")],
-    ) as mock_match:
-        id1 = _persist_house(db, h1, "https://avito.ru/house/111")
-        id2 = _persist_house(db, h2, "https://avito.ru/house/222")
-
-    assert id1 == 42, f"Первый вызов должен вернуть 42, получили {id1}"
-    assert id2 == 42, f"Второй вызов должен вернуть тот же 42, получили {id2}"
-    assert id1 == id2, "Оба вызова должны вернуть один house_id (нет дубля)"
-    assert mock_match.call_count == 2
-
-
-# ---------------------------------------------------------------------------
-# Test 2: enrichment-поля записываются в UPDATE
-# ---------------------------------------------------------------------------
-
-
-def test_persist_house_enrichment_fields_written():
-    """Enrichment-поля (rating_score, developer_name) передаются в UPDATE-запрос."""
-    db = MagicMock()
-    db.execute.return_value = MagicMock()
-
-    h = _make_house(ext_id=111)
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-        return_value=(42, 1.0, "source_exact"),
-    ):
-        _persist_house(db, h, "https://avito.ru/house/111")
-
-    # db.execute должен быть вызван 1 раз (UPDATE houses SET ...)
-    assert (
-        db.execute.call_count == 1
-    ), f"Ожидался 1 вызов db.execute (UPDATE), получили {db.execute.call_count}"
-
-    # Проверяем параметры bind
-    call_args = db.execute.call_args
-    sql_obj = call_args[0][0]
-    bind = call_args[0][1]
-
-    assert "UPDATE houses" in str(sql_obj), "SQL должен содержать UPDATE houses"
-    assert bind["rating_score"] == 4.5, f"rating_score={bind.get('rating_score')!r}"
-    assert bind["developer_name"] == "ПИК", f"developer_name={bind.get('developer_name')!r}"
-    assert bind["house_id"] == 42, f"house_id={bind.get('house_id')!r}"
-
-    # avito_id_hash должен быть передан
-    assert bind["avito_id_hash"] == "hash_111", f"avito_id_hash={bind.get('avito_id_hash')!r}"
-
-    # rating_distribution должен быть JSON-строкой
-    rd = bind.get("rating_distribution")
-    assert isinstance(rd, str), f"rating_distribution должен быть str (JSON), получили {type(rd)}"
-    assert json.loads(rd) == [{"score": 5, "count": 80}]
-
-
-# ---------------------------------------------------------------------------
-# Test 3: _persist_house не вызывает db.commit()
-# ---------------------------------------------------------------------------
-
-
-def test_persist_house_no_commit():
-    """_persist_house не должен вызывать db.commit() — коммитит вызывающий."""
-    db = MagicMock()
-    db.execute.return_value = MagicMock()
-
-    h = _make_house()
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-        return_value=(7, 1.0, "new"),
-    ):
-        _persist_house(db, h, "")
-
-    db.commit.assert_not_called()
-
-
-# ---------------------------------------------------------------------------
-# Test 4: upsert_house делегирует в _persist_house (с пустым url)
-# ---------------------------------------------------------------------------
-
-
-def test_upsert_house_delegates_to_persist_house():
-    """upsert_house(db, h) → _persist_house(db, h, "") → возвращает house_id."""
-    db = MagicMock()
-    db.execute.return_value = MagicMock()
-    h = _make_house()
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-        return_value=(55, 1.0, "new"),
-    ):
-        result = upsert_house(db, h)
-
-    assert result == 55
-
-    # Проверяем что UPDATE был вызван с url="" (пустая строка — нет url в этом пути)
-    bind = db.execute.call_args[0][1]
-    assert bind["url"] == "", f"url должен быть пустой строкой, получили {bind.get('url')!r}"
-
-
-# ---------------------------------------------------------------------------
-# Test 5: _upsert_house_with_url делегирует в _persist_house (с реальным url)
-# ---------------------------------------------------------------------------
-
-
-def test_upsert_house_with_url_delegates_to_persist_house():
-    """_upsert_house_with_url(db, h, url) → _persist_house(db, h, url) → house_id."""
-    db = MagicMock()
-    db.execute.return_value = MagicMock()
-    h = _make_house()
-    url = "https://www.avito.ru/catalog/houses/ekb/dom-1234"
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-        return_value=(99, 1.0, "new"),
-    ):
-        result = _upsert_house_with_url(db, h, url)
-
-    assert result == 99
-
-    # Проверяем что правильный url передан в UPDATE
-    bind = db.execute.call_args[0][1]
-    assert bind["url"] == url, f"url должен быть {url!r}, получили {bind.get('url')!r}"
-
-
-# ---------------------------------------------------------------------------
-# Test 6: match_or_create_house вызывается с правильными аргументами
-# ---------------------------------------------------------------------------
-
-
-def test_persist_house_calls_match_with_correct_args():
-    """_persist_house передаёт правильные аргументы в match_or_create_house."""
-    db = MagicMock()
-    db.execute.return_value = MagicMock()
-
-    h = _make_house(ext_id=333)
-    url = "https://avito.ru/333"
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-        return_value=(10, 1.0, "new"),
-    ) as mock_match:
-        _persist_house(db, h, url)
-
-    mock_match.assert_called_once_with(
-        db,
-        ext_source="avito",
-        ext_id="333",
-        address=h.short_address or h.full_address,
-        lat=h.lat,
-        lon=h.lon,
-        year_built=h.year_built,
-        source_url=url,
-    )
-
-
-# ---------------------------------------------------------------------------
-# Test 7: COALESCE — адрес/координаты не затираются через UPDATE параметры
-#   (проверяем что bind содержит address/lat/lon для COALESCE в SQL)
-# ---------------------------------------------------------------------------
-
-
-def test_persist_house_coalesce_fields_in_bind():
-    """UPDATE передаёт address/lat/lon в bind для COALESCE(houses.<col>, CAST(:x AS ...))."""
-    db = MagicMock()
-    db.execute.return_value = MagicMock()
-
-    h = _make_house()
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-        return_value=(1, 1.0, "new"),
-    ):
-        _persist_house(db, h, "")
-
-    bind = db.execute.call_args[0][1]
-    assert bind["address"] == h.short_address
-    assert bind["lat"] == h.lat
-    assert bind["lon"] == h.lon
-    assert bind["year_built"] == h.year_built
-
-
-# ---------------------------------------------------------------------------
-# Test 7b: #2003 — house_class передаётся в UPDATE bind (COALESCE fill-only)
-# ---------------------------------------------------------------------------
-
-
-def test_persist_house_house_class_in_bind():
-    """UPDATE передаёт house_class в bind для COALESCE(CAST(:house_class AS text), houses.*)."""
-    db = MagicMock()
-    db.execute.return_value = MagicMock()
-
-    h = _make_house()
-    h.house_class = "comfort"
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-        return_value=(1, 1.0, "new"),
-    ):
-        _persist_house(db, h, "")
-
-    bind = db.execute.call_args[0][1]
-    assert bind["house_class"] == "comfort"
-
-    sql_obj = db.execute.call_args[0][0]
-    assert "house_class" in str(sql_obj)
-
-
-def test_persist_house_house_class_none_does_not_null_via_coalesce():
-    """house_class=None в bind — SQL использует COALESCE(new, existing), не голый overwrite."""
-    db = MagicMock()
-    db.execute.return_value = MagicMock()
-
-    h = _make_house()  # house_class остаётся дефолтным None
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-        return_value=(1, 1.0, "new"),
-    ):
-        _persist_house(db, h, "")
-
-    bind = db.execute.call_args[0][1]
-    assert bind["house_class"] is None
-
-    sql_obj = str(db.execute.call_args[0][0])
-    # SQL должен явно защищать существующее значение через COALESCE, а не
-    # безусловный overwrite (который занулил бы значение от другого источника).
-    assert "COALESCE(CAST(:house_class AS text), houses.house_class)" in sql_obj
-
-
-# ---------------------------------------------------------------------------
-# Test 8: #1820 — ext_id=0 graceful-skip гард
-#   Два id-less URL (ext_id=0, без адреса) → match_or_create_house НЕ вызывается,
-#   house_sources("avito","0") НЕ создаётся, возвращается sentinel dict с house_id=0.
-# ---------------------------------------------------------------------------
-
-
-def test_save_house_catalog_enrichment_skips_zero_ext_id_no_address():
-    """#1820: ext_id=0 + нет адреса → гард пропускает персистирование canonical house."""
-    db = MagicMock()
-
-    # Создаём два id-less graceful-skip enrichment (как _parse_development_or_skip возвращает)
-    enrichment_a = HouseCatalogEnrichment(
-        house_url="https://www.avito.ru/catalog/novostroyki/dom-A/",
-        house=HouseInfo(ext_id=0),
-    )
-    enrichment_b = HouseCatalogEnrichment(
-        house_url="https://www.avito.ru/catalog/novostroyki/dom-B/",
-        house=HouseInfo(ext_id=0),
-    )
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-    ) as mock_match:
-        result_a = save_house_catalog_enrichment(db, enrichment_a)
-        result_b = save_house_catalog_enrichment(db, enrichment_b)
-
-    # match_or_create_house НЕ должен вызываться — гард сработал до него
-    mock_match.assert_not_called()
-
-    # Возвращается sentinel dict с house_id=0, не плодятся дубли
-    assert result_a["house_id"] == 0, f"ожидали house_id=0, получили {result_a['house_id']}"
-    assert result_b["house_id"] == 0, f"ожидали house_id=0, получили {result_b['house_id']}"
-
-    # db.commit() НЕ вызывается (нет реальной транзакции)
-    db.commit.assert_not_called()
-
-
-def test_save_house_catalog_enrichment_with_address_persists_despite_zero_ext_id():
-    """#1820: ext_id=0 НО есть адрес → НЕ пропускаем (developmentPage с адресом, без avitoId)."""
-    db = MagicMock()
-    db.execute.return_value = MagicMock()
-
-    enrichment = HouseCatalogEnrichment(
-        house_url="https://www.avito.ru/catalog/novostroyki/dom-C/",
-        house=HouseInfo(ext_id=0, short_address="ул. Ленина, 1"),
-    )
-
-    with patch(
-        "app.services.scrapers.avito_houses.match_or_create_house",
-        return_value=(77, 0.9, "fingerprint"),
-    ) as mock_match:
-        result = save_house_catalog_enrichment(db, enrichment)
-
-    # Адрес есть → персистим через fingerprint tier
-    mock_match.assert_called_once()
-    assert result["house_id"] == 77

tradein-mvp/backend/tests/test_avito_houses_parse.py

@@ -1,790 +0,0 @@
-"""Offline smoke for avito_houses parser.
-
-Uses minimal JSON fixture mimicking Avito __preloadedState__ structure.
-Не требует сети, БД или curl_cffi — тестирует только чистый парсер.
-"""
-
-from datetime import date
-from pathlib import Path
-
-import pytest
-
-from app.services.scrapers.avito_houses import (
-    HouseCatalogEnrichment,
-    HouseInfo,
-    HouseReview,
-    MiniSerpListing,
-    PlacementHistoryItem,
-    RecommendationItem,
-    SellerInfo,
-    _extract_preloaded_state,
-    _parse_ru_date,
-    parse_houses_state,
-)
-
-FIXTURES_DIR = Path(__file__).parent / "fixtures"
-
-
-# ---------------------------------------------------------------------------
-# Minimal __preloadedState__ fixture (Python dict — не raw HTML)
-# ---------------------------------------------------------------------------
-
-MINIMAL_STATE: dict = {
-    "data": {
-        "data": {
-            "page": {
-                "placeholders": [
-                    {"type": "breadcrumbs"},
-                    {"type": "title"},
-                    {"type": "gallery", "props": {}},
-                    {
-                        "type": "housePage",
-                        "props": {
-                            "developmentData": {
-                                "avitoId": 3171365,
-                                "id": "MTE3LjM3MDg5MA",
-                                "title": "Akademika Postovskogo 17a",
-                                "address": "ул. Постовского, 17а",
-                                "fullAddress": (
-                                    "Свердловская обл., Екатеринбург, ул. Постовского, 17а"
-                                ),
-                                "coords": {"lat": 56.790699, "lng": 60.580191},
-                                "aboutDevelopment": {
-                                    "expandParams": {
-                                        "items": [
-                                            {
-                                                "title": "Параметры дома",
-                                                "params": [
-                                                    {"type": "Год постройки", "value": "2019"},
-                                                    {"type": "Этажей", "value": "25"},
-                                                    {"type": "Тип дома", "value": "Монолитный"},
-                                                    {"type": "Пассажирский лифт", "value": "2"},
-                                                    {"type": "Консьерж", "value": "Да"},
-                                                    {"type": "Закрытый двор", "value": "Да"},
-                                                    {"type": "Детская площадка", "value": "Да"},
-                                                ],
-                                            },
-                                        ]
-                                    }
-                                },
-                                "developer": {"name": "Атомстройкомплекс", "key": "atomstroy"},
-                                "mapPreview": {
-                                    "distance": "в 5 минутах ходьбы",
-                                    "objects": "школа, парк, аптека",
-                                    "pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}],
-                                },
-                                "ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}},
-                                "ratingSummaryStat": {
-                                    "ratingStat": [
-                                        {"score": 5, "count": 4},
-                                        {"score": 4, "count": 2},
-                                    ],
-                                    "reviewCount": 6,
-                                },
-                            }
-                        },
-                    },
-                    {
-                        "type": "reviews",
-                        "props": {
-                            "entries": [
-                                {
-                                    "type": "rating",
-                                    "value": {
-                                        "id": 12345,
-                                        "author": {"title": "Иван"},
-                                        "reviewTitle": "Хорошо",
-                                        "score": 5,
-                                        "modelExperience": "Живу в своей квартире",
-                                        "rated": "9 октября 2025",
-                                        "textSections": [
-                                            {"title": "", "text": "Общий отзыв"},
-                                            {"title": "Преимущества", "text": "Парк рядом"},
-                                            {"title": "Недостатки", "text": "Парковки мало"},
-                                        ],
-                                    },
-                                },
-                                {"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}},
-                            ]
-                        },
-                    },
-                    {
-                        "type": "miniSerp",
-                        "props": {
-                            "items": [
-                                {
-                                    "id": 7986882804,
-                                    "title": "3-к. квартира",
-                                    "description": "Описание",
-                                    "url": "/ekaterinburg/kvartiry/test_7986882804",
-                                    "price": "11 990 000 ₽",
-                                    "geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]},
-                                    "seller": {
-                                        "name": "DOMRF66",
-                                        "type": "Компания",
-                                        "from": "На Авито с июля 2012",
-                                        "url": "/brands/domrf66",
-                                        "logos": [],
-                                    },
-                                },
-                            ]
-                        },
-                    },
-                    {
-                        "type": "housePlacementHistory",
-                        "props": {
-                            "items": [
-                                {
-                                    "id": 7000000001,
-                                    "title": "1-к. квартира",
-                                    "startPrice": 5000000,
-                                    "startPriceDate": 1697000000,
-                                    "lastPrice": 4900000,
-                                    "lastPriceDate": 1715000000,
-                                    "exposure": 210,
-                                    "itemImage": {},
-                                },
-                            ]
-                        },
-                    },
-                    {"type": "uxFeedback"},
-                    {
-                        "type": "recommendations",
-                        "props": {
-                            "items": [
-                                {
-                                    "title": "Похожая",
-                                    "priceRange": "от 6 млн",
-                                    "additionalInfo": ["Адрес 1"],
-                                    "url": "/rec1",
-                                    "images": [{"208x156": "https://avito.ru/img1.jpg"}],
-                                },
-                            ]
-                        },
-                    },
-                    {"type": "padding"},
-                ]
-            }
-        }
-    }
-}
-
-
-# ---------------------------------------------------------------------------
-# Helpers
-# ---------------------------------------------------------------------------
-
-
-def test_ru_date_parse() -> None:
-    assert _parse_ru_date("9 октября 2025") == date(2025, 10, 9)
-    assert _parse_ru_date("31 декабря 2020") == date(2020, 12, 31)
-    assert _parse_ru_date("1 января 2000") == date(2000, 1, 1)
-    assert _parse_ru_date(None) is None
-    assert _parse_ru_date("bogus") is None
-    assert _parse_ru_date("") is None
-
-
-# ---------------------------------------------------------------------------
-# Main parse test
-# ---------------------------------------------------------------------------
-
-
-def test_parse_minimal_state() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-
-    assert isinstance(e, HouseCatalogEnrichment)
-    assert e.house_url == "/catalog/houses/ekaterinburg/test/3171365"
-
-
-def test_house_info() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-    h = e.house
-
-    assert isinstance(h, HouseInfo)
-    assert h.ext_id == 3171365
-    assert h.ext_id_hash == "MTE3LjM3MDg5MA"
-    assert h.title == "Akademika Postovskogo 17a"
-    assert h.short_address == "ул. Постовского, 17а"
-    assert h.full_address == "Свердловская обл., Екатеринбург, ул. Постовского, 17а"
-    assert h.lat == 56.790699
-    assert h.lon == 60.580191
-
-
-def test_house_expand_params() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-    h = e.house
-
-    assert h.year_built == 2019
-    assert h.total_floors == 25
-    assert h.house_type == "monolith"
-    assert h.passenger_elevators == 2
-    assert h.has_concierge is True
-    assert h.closed_yard is True
-    assert h.has_playground is True
-
-
-def test_house_developer_and_rating() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-    h = e.house
-
-    assert h.developer_name == "Атомстройкомплекс"
-    assert h.developer_key == "atomstroy"
-    assert h.rating_score == 4.666
-    assert h.rating_string == "4,7"
-    assert h.reviews_count == 6
-    assert len(h.rating_distribution) == 2
-
-
-def test_house_map_and_infra() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-    h = e.house
-
-    assert h.infrastructure_walk_distance == "в 5 минутах ходьбы"
-    assert h.infrastructure_summary == "школа, парк, аптека"
-    assert len(h.map_pins) == 1
-    assert h.map_pins[0]["type"] == "school"
-
-
-def test_house_raw_characteristics() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-    # raw_characteristics = полный expandParams items (1 категория)
-    assert len(e.house.raw_characteristics) == 1
-    assert e.house.raw_characteristics[0]["title"] == "Параметры дома"
-
-
-def test_reviews() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-
-    assert len(e.reviews) == 1
-    assert e.reviews_next_page_url == "/reviews?page=2"
-
-    r = e.reviews[0]
-    assert isinstance(r, HouseReview)
-    assert r.ext_review_id == 12345
-    assert r.author_name == "Иван"
-    assert r.review_title == "Хорошо"
-    assert r.score == 5
-    assert r.model_experience == "Живу в своей квартире"
-    assert r.rated_date == date(2025, 10, 9)
-    assert r.text_main == "Общий отзыв"
-    assert r.text_pros == "Парк рядом"
-    assert r.text_cons == "Парковки мало"
-    assert r.raw_payload is not None
-
-
-def test_mini_serp() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-
-    assert len(e.mini_serp) == 1
-    m = e.mini_serp[0]
-    assert isinstance(m, MiniSerpListing)
-    assert m.ext_item_id == 7986882804
-    assert m.price_rub == 11990000
-    assert m.metro_text == "Чкаловская, 2,7 км"
-    assert m.metro_color == "#cf2734"
-    assert m.description == "Описание"
-
-
-def test_seller_extraction() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-    m = e.mini_serp[0]
-
-    assert m.seller is not None
-    s = m.seller
-    assert isinstance(s, SellerInfo)
-    assert s.ext_seller_id == "domrf66"
-    assert s.name == "DOMRF66"
-    assert s.seller_type == "Компания"
-    assert s.on_platform_since == "На Авито с июля 2012"
-    assert s.profile_url == "/brands/domrf66"
-
-
-def test_placement_history() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-
-    assert len(e.placement_history) == 1
-    p = e.placement_history[0]
-    assert isinstance(p, PlacementHistoryItem)
-    assert p.ext_item_id == "7000000001"
-    assert p.title == "1-к. квартира"
-    assert p.start_price == 5000000
-    assert p.last_price == 4900000
-    assert p.exposure_days == 210
-    # removed_date ВСЕГДА None — виджет не отдаёт!
-    assert p.removed_date is None
-    # raw_payload не содержит itemImage (скипается)
-    assert p.raw_payload is not None
-    assert "itemImage" not in p.raw_payload
-    # start_price_date и last_price_date — из unix timestamp
-    assert isinstance(p.start_price_date, date)
-    assert isinstance(p.last_price_date, date)
-
-
-def test_recommendations() -> None:
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-
-    assert len(e.recommendations) == 1
-    rec = e.recommendations[0]
-    assert isinstance(rec, RecommendationItem)
-    assert rec.title == "Похожая"
-    assert rec.price_text == "от 6 млн"
-    assert rec.address == "Адрес 1"
-    assert rec.url == "/rec1"
-    assert rec.image_url == "https://avito.ru/img1.jpg"
-
-
-def test_missing_optional_widgets() -> None:
-    """Парсер не падает если опциональные виджеты отсутствуют."""
-    state_minimal: dict = {
-        "data": {
-            "data": {
-                "page": {
-                    "placeholders": [
-                        {
-                            "type": "housePage",
-                            "props": {
-                                "developmentData": {
-                                    "avitoId": 9999,
-                                    "coords": {"lat": 56.0, "lng": 60.0},
-                                    "aboutDevelopment": {"expandParams": {"items": []}},
-                                    "developer": {},
-                                    "mapPreview": {},
-                                    "ratingBadge": {"info": {}},
-                                    "ratingSummaryStat": {},
-                                }
-                            },
-                        },
-                    ]
-                }
-            }
-        }
-    }
-    e = parse_houses_state(state_minimal, "/catalog/houses/test/9999")
-    assert e.house.ext_id == 9999
-    assert e.reviews == []
-    assert e.mini_serp == []
-    assert e.placement_history == []
-    assert e.recommendations == []
-    assert e.reviews_next_page_url is None
-
-
-def test_invalid_state_raises() -> None:
-    """ValueError если placeholders не найден."""
-    with pytest.raises(ValueError, match="placeholders"):
-        parse_houses_state({}, "/catalog/houses/test/1")
-
-
-def test_missing_house_page_raises() -> None:
-    """ValueError если housePage виджет отсутствует (вторичка — /catalog/houses/)."""
-    state: dict = {
-        "data": {
-            "data": {
-                "page": {
-                    "placeholders": [
-                        {"type": "breadcrumbs"},
-                    ]
-                }
-            }
-        }
-    }
-    with pytest.raises(ValueError, match="housePage"):
-        parse_houses_state(state, "/catalog/houses/test/1")
-
-
-def test_novostroyka_minimal_development_page() -> None:
-    """Новостройка: виджет 'developmentPage' парсится в partial HouseInfo без ValueError."""
-    state: dict = {
-        "data": {
-            "data": {
-                "page": {
-                    "placeholders": [
-                        {"type": "reNavigation"},
-                        {
-                            "type": "developmentPage",
-                            "props": {
-                                "developmentPage": {
-                                    "developmentData": {
-                                        "avitoId": 12879380,
-                                        "id": "MGDKHN70PQ",
-                                        "title": "ЖК Береговой",
-                                        "address": "ул. Щербакова, жилые дома",
-                                        "fullAddress": "",
-                                        "coords": {"lat": 56.77005, "lng": 60.67244},
-                                        "developer": {"name": "Талан", "key": "talan-1"},
-                                        "mapPreview": {
-                                            "distance": "в 15 минутах ходьбы",
-                                            "objects": "школа, детский сад",
-                                            "pins": [{"lat": 56.77, "lng": 60.67}],
-                                        },
-                                        "aboutDevelopment": {
-                                            "tabs": [
-                                                {
-                                                    "title": "Параметры",
-                                                    "expandParams": {
-                                                        "items": [
-                                                            {
-                                                                "params": [
-                                                                    {
-                                                                        "type": "Тип дома",
-                                                                        "value": "Монолит",
-                                                                    },
-                                                                    {
-                                                                        "type": "Этажность",
-                                                                        "value": "от 4 до 24",
-                                                                    },
-                                                                    {
-                                                                        "type": (
-                                                                            "Пассажирских лифтов"
-                                                                        ),
-                                                                        "value": "42",
-                                                                    },
-                                                                    {
-                                                                        "type": "Грузовых лифтов",
-                                                                        "value": "60",
-                                                                    },
-                                                                ]
-                                                            }
-                                                        ]
-                                                    },
-                                                }
-                                            ]
-                                        },
-                                    }
-                                }
-                            },
-                        },
-                        {"type": "uxFeedback"},
-                    ]
-                }
-            }
-        }
-    }
-    url = "/catalog/novostroyki/ekaterinburg/zhk_beregovoy/12879380"
-    e = parse_houses_state(state, url)
-
-    assert isinstance(e, HouseCatalogEnrichment)
-    assert e.house_url == url
-    h = e.house
-    assert h.ext_id == 12879380
-    assert h.ext_id_hash == "MGDKHN70PQ"
-    assert h.title == "ЖК Береговой"
-    assert h.short_address == "ул. Щербакова, жилые дома"
-    assert h.lat == 56.77005
-    assert h.lon == 60.67244
-    assert h.house_type == "monolith"  # "Монолит" → monolith
-    assert h.passenger_elevators == 42
-    assert h.cargo_elevators == 60
-    # Этажность = диапазон "от 4 до 24" → int-каст None (только в raw_characteristics)
-    assert h.total_floors is None
-    assert h.year_built is None  # дом строится, года постройки нет
-    assert h.developer_name == "Талан"
-    assert h.developer_key == "talan-1"
-    assert h.infrastructure_summary == "школа, детский сад"
-    assert h.infrastructure_walk_distance == "в 15 минутах ходьбы"
-    assert len(h.map_pins) == 1
-    # Нет рейтинга/отзывов/истории у новостройки
-    assert h.rating_score is None
-    assert e.reviews == []
-    assert e.placement_history == []
-    assert e.recommendations == []
-
-
-def test_novostroyka_no_development_page_graceful_skip() -> None:
-    """Новостройка без developmentPage/housePage → graceful skip (НЕ ValueError)."""
-    state: dict = {
-        "data": {
-            "data": {
-                "page": {
-                    "placeholders": [
-                        {"type": "reNavigation"},
-                        {"type": "uxFeedback"},
-                    ]
-                }
-            }
-        }
-    }
-    url = "/catalog/novostroyki/ekaterinburg/zhk_unknown/55667788"
-    e = parse_houses_state(state, url)
-
-    assert isinstance(e, HouseCatalogEnrichment)
-    assert e.house_url == url
-    # ext_id извлечён из хвоста URL
-    assert e.house.ext_id == 55667788
-    assert e.reviews == []
-    assert e.placement_history == []
-
-
-def test_novostroyka_real_fixture_end_to_end() -> None:
-    """Реальная фикстура новостройки: parse не кидает ValueError, отдаёт partial-обогащение."""
-    html = (FIXTURES_DIR / "avito_novostroyka_12879380.html").read_text(encoding="utf-8")
-    state = _extract_preloaded_state(html)
-    assert state is not None, "URL-encoded __preloadedState__ должен распарситься"
-
-    url = "/catalog/novostroyki/ekaterinburg/semeynyy_kvartal_beregovoy/12879380"
-    e = parse_houses_state(state, url)
-
-    assert isinstance(e, HouseCatalogEnrichment)
-    h = e.house
-    assert h.ext_id == 12879380
-    assert h.ext_id_hash  # base64 internal id
-    assert h.house_type == "monolith"
-    assert h.passenger_elevators is not None and h.passenger_elevators > 0
-    assert h.lat is not None and h.lon is not None
-    assert h.developer_name  # застройщик присутствует
-    # Новостройка: нет отзывов/истории/года постройки
-    assert h.year_built is None
-    assert e.reviews == []
-    assert e.placement_history == []
-
-
-def test_house_type_normalization() -> None:
-    """Все варианты типа дома нормализуются корректно."""
-    from app.services.scrapers.avito_houses import _normalize_house_type
-
-    assert _normalize_house_type("Монолитный") == "monolith"
-    assert _normalize_house_type("Панельный") == "panel"
-    assert _normalize_house_type("Кирпичный") == "brick"
-    assert _normalize_house_type("Монолитно-кирпичный") == "monolith_brick"
-    assert _normalize_house_type("Блочный") == "block"
-    assert _normalize_house_type("Деревянный") == "wood"
-    assert _normalize_house_type("Неизвестный") == "other"
-    assert _normalize_house_type(None) is None
-
-
-def test_house_class_normalization() -> None:
-    """Все варианты класса жилья нормализуются корректно (#2003)."""
-    from app.services.scrapers.avito_houses import _normalize_house_class
-
-    assert _normalize_house_class("Комфорт") == "comfort"
-    assert _normalize_house_class("Бизнес") == "business"
-    assert _normalize_house_class("Премиум") == "premium"
-    assert _normalize_house_class("Эконом") == "economy"
-    assert _normalize_house_class("Комфорт+") == "comfort_plus"
-    assert _normalize_house_class("Элит") == "elite"
-    # Регистронезависимость + пробелы по краям
-    assert _normalize_house_class("  комфорт  ") == "comfort"
-    # Неизвестное значение → None (НЕ "other" — house_class точный ANY(...) фильтр)
-    assert _normalize_house_class("Неизвестный класс") is None
-    assert _normalize_house_class(None) is None
-    assert _normalize_house_class("") is None
-
-
-def test_house_class_parsed_from_novostroyka_expand_params() -> None:
-    """ "Класс жилья" из developmentPage expandParams → HouseInfo.house_class (#2003)."""
-    state: dict = {
-        "data": {
-            "data": {
-                "page": {
-                    "placeholders": [
-                        {
-                            "type": "developmentPage",
-                            "props": {
-                                "developmentPage": {
-                                    "developmentData": {
-                                        "avitoId": 42,
-                                        "coords": {"lat": 56.8, "lng": 60.6},
-                                        "developer": {},
-                                        "mapPreview": {},
-                                        "aboutDevelopment": {
-                                            "tabs": [
-                                                {
-                                                    "expandParams": {
-                                                        "items": [
-                                                            {
-                                                                "params": [
-                                                                    {
-                                                                        "type": "Класс жилья",
-                                                                        "value": "Комфорт",
-                                                                    },
-                                                                ]
-                                                            }
-                                                        ]
-                                                    }
-                                                }
-                                            ]
-                                        },
-                                    }
-                                }
-                            },
-                        },
-                    ]
-                }
-            }
-        }
-    }
-    e = parse_houses_state(state, "/catalog/novostroyki/ekaterinburg/test/42")
-    assert e.house.house_class == "comfort"
-
-
-def test_house_class_absent_on_secondary_listing() -> None:
-    """Вторичка (housePage) не отдаёт "Класс жилья" → house_class остаётся None."""
-    e = parse_houses_state(MINIMAL_STATE, "/catalog/houses/ekaterinburg/test/3171365")
-    assert e.house.house_class is None
-
-
-def test_strip_price() -> None:
-    """_strip_price корректно извлекает числа из строк Avito."""
-    from app.services.scrapers.avito_houses import _strip_price
-
-    assert _strip_price("11 990 000 ₽") == 11990000
-    assert _strip_price("5 000 000 ₽") == 5000000
-    assert _strip_price(None) is None
-    assert _strip_price("") is None
-
-
-def test_mini_serp_no_seller() -> None:
-    """Объявление без продавца не вызывает ошибку."""
-    state: dict = {
-        "data": {
-            "data": {
-                "page": {
-                    "placeholders": [
-                        {
-                            "type": "housePage",
-                            "props": {
-                                "developmentData": {
-                                    "avitoId": 111,
-                                    "coords": {"lat": 56.0, "lng": 60.0},
-                                    "aboutDevelopment": {"expandParams": {"items": []}},
-                                    "developer": {},
-                                    "mapPreview": {},
-                                    "ratingBadge": {"info": {}},
-                                    "ratingSummaryStat": {},
-                                }
-                            },
-                        },
-                        {
-                            "type": "miniSerp",
-                            "props": {
-                                "items": [
-                                    {
-                                        "id": 12345,
-                                        "title": "1-к.",
-                                        "url": "/test",
-                                        "price": "3 000 000 ₽",
-                                        "geo": {"content": "Центр", "colors": []},
-                                    },
-                                ]
-                            },
-                        },
-                    ]
-                }
-            }
-        }
-    }
-    e = parse_houses_state(state, "/catalog/houses/test/111")
-    assert len(e.mini_serp) == 1
-    assert e.mini_serp[0].seller is None
-    assert e.mini_serp[0].price_rub == 3000000
-
-
-# ---------------------------------------------------------------------------
-# Real-HTML fixture (regression for issue #566 — state-blob encoding/shape change)
-# ---------------------------------------------------------------------------
-#
-# Fixture: реальная страница /catalog/houses/ekaterinburg/ul_yumasheva_6/115188,
-# захвачена curl_cffi chrome120 (2026-05-28). Avito сменил формат __preloadedState__
-# на URL-encoded литерал + обернул виджеты в placeholders[0]['widgets'] +
-# вложил developmentData под props['housePage']. Этот тест ловит регрессию декода/shape.
-
-
-def test_extract_preloaded_state_from_real_html() -> None:
-    """_extract_preloaded_state декодирует текущий URL-encoded __preloadedState__."""
-    html = (FIXTURES_DIR / "avito_houses_sample.html").read_text(encoding="utf-8")
-
-    state = _extract_preloaded_state(html)
-
-    assert state is not None, "URL-encoded __preloadedState__ должен распарситься"
-    # Avito SSR обёртка — успешный ответ
-    assert state["data"]["status"] == 200
-    # Inner path до placeholders сохранился
-    placeholders = state["data"]["data"]["page"]["placeholders"]
-    assert isinstance(placeholders, list) and placeholders, "placeholders непустой массив"
-
-
-def test_parse_real_fixture_end_to_end() -> None:
-    """Фикстура HTML → _extract_preloaded_state → parse_houses_state → HouseInfo.ext_id."""
-    html = (FIXTURES_DIR / "avito_houses_sample.html").read_text(encoding="utf-8")
-    state = _extract_preloaded_state(html)
-    assert state is not None
-
-    e = parse_houses_state(state, "/catalog/houses/ekaterinburg/ul_yumasheva_6/115188")
-
-    assert isinstance(e, HouseCatalogEnrichment)
-    # housePage распарсился — ext_id (avitoId) присутствует и совпадает с URL
-    assert e.house.ext_id == 115188
-    assert e.house.ext_id_hash  # base64 internal id присутствует
-    # expandParams распарсились (год постройки реальной фикстуры — 2016)
-    assert e.house.year_built == 2016
-    assert e.house.total_floors == 17
-    assert e.house.house_type == "monolith"
-    assert e.house.lat is not None and e.house.lon is not None
-
-
-def test_extract_preloaded_state_jsparse_fallback() -> None:
-    """Старая форма JSON.parse("...") всё ещё декодируется (fallback-ветка)."""
-    inner = '{\\"data\\":{\\"data\\":{\\"page\\":{\\"placeholders\\":[]}}}}'
-    html = f'<script>window.__preloadedState__ = JSON.parse("{inner}");</script>'
-
-    state = _extract_preloaded_state(html)
-
-    assert state is not None
-    assert state["data"]["data"]["page"]["placeholders"] == []
-
-
-def test_extract_preloaded_state_missing_returns_none() -> None:
-    """Нет маркера → None (а не исключение)."""
-    assert _extract_preloaded_state("<html><body>no state here</body></html>") is None
-
-
-# ---------------------------------------------------------------------------
-# Real-HTML fixture (regression for issue #1789 — MFE props nesting)
-# ---------------------------------------------------------------------------
-#
-# Fixture: реальная страница /catalog/houses/ekaterinburg/ul_soni_morozovoy_190/572538,
-# захвачена браузер-сервисом (2026-06-19). Avito перенёс данные виджетов на уровень
-# глубже — под ключ с именем виджета:
-#   reviews:               props['reviews']['entries']   (было props['entries'])
-#   miniSerp:              props['miniSerp']['items']    (было props['items'])
-#   housePlacementHistory: props['housePlacementHistory']['items'] (было props['items'])
-#   recommendations:       props['recommendations']['items']       (было props['items'])
-# Из-за этого reviews/placement/seller-обогащение НЕ долетало в БД (0 строк).
-# Этот тест ловит регрессию шейпа props.
-
-
-def test_parse_mfe_fixture_widgets_reach_db() -> None:
-    """Фикстура MFE: reviews/placement/miniSerp/house_type/year_built распарсены непусто."""
-    html = (FIXTURES_DIR / "avito_house_572538.html").read_text(encoding="utf-8")
-    state = _extract_preloaded_state(html)
-    assert state is not None, "URL-encoded __preloadedState__ должен распарситься"
-
-    e = parse_houses_state(state, "/catalog/houses/ekaterinburg/ul_soni_morozovoy_190/572538")
-
-    # housePage — тип дома и год постройки долетают
-    assert e.house.ext_id == 572538
-    assert e.house.house_type is not None, "house_type должен парситься из expandParams"
-    assert e.house.house_type == "brick"
-    assert e.house.year_built == 1994
-    assert e.house.reviews_count == 3
-
-    # reviews — >=1 отзыв с текстом, автором, оценкой
-    assert len(e.reviews) >= 1, "reviews должны распарситься из props['reviews']['entries']"
-    r = e.reviews[0]
-    assert r.ext_review_id, "у отзыва должен быть id"
-    assert r.author_name, "у отзыва должен быть автор"
-    assert r.score is not None, "у отзыва должна быть оценка"
-    assert r.text_main, "у отзыва должен быть основной текст (textSections)"
-
-    # placement — >=1 item из props['housePlacementHistory']['items']
-    assert len(e.placement_history) >= 1, "placement должен распарситься"
-    p = e.placement_history[0]
-    assert p.ext_item_id, "у placement item должен быть id"
-    assert p.start_price is not None
-    assert p.last_price is not None
-    assert isinstance(p.last_price_date, date)
-
-    # miniSerp — >=1 объявление из props['miniSerp']['items']
-    assert len(e.mini_serp) >= 1, "miniSerp должен распарситься"
-    m = e.mini_serp[0]
-    assert m.ext_item_id
-    assert m.price_rub is not None

tradein-mvp/backend/tests/test_backfill_wave2.py

@@ -234,9 +234,10 @@ async def test_backfill_yandex_addresses_error_on_non_200():
 @pytest.mark.asyncio
 async def test_cian_price_history_saves_changes():
     """Happy path: fetch returns DetailEnrichment with price_changes → saved."""
+    from scraper_kit.providers.cian.detail import DetailEnrichment
+
     from app.services.cian_price_history import backfill_cian_price_history
     from app.services.scraper_adapters import RealScraperConfig
-    from app.services.scrapers.cian_detail import DetailEnrichment
 
     enrichment = DetailEnrichment(
         cian_id=123456,
@@ -290,9 +291,10 @@ async def test_cian_price_history_saves_changes():
 @pytest.mark.asyncio
 async def test_cian_price_history_skips_no_changes():
     """Fetch succeeds but no price_changes → skipped (not error)."""
+    from scraper_kit.providers.cian.detail import DetailEnrichment
+
     from app.services.cian_price_history import backfill_cian_price_history
     from app.services.scraper_adapters import RealScraperConfig
-    from app.services.scrapers.cian_detail import DetailEnrichment
 
     enrichment = DetailEnrichment(cian_id=111, price_changes=[])
 

tradein-mvp/backend/tests/test_cian_detail.py

@@ -1,830 +0,0 @@
-"""Tests for cian_detail.py — Stage 5 of CianScraper v1."""
-
-from __future__ import annotations
-
-from pathlib import Path
-from unittest.mock import AsyncMock, MagicMock
-
-import pytest
-
-from app.services.scrapers.cian_detail import (
-    DetailEnrichment,
-    _extract_agent_profile,
-    _extract_ceiling_height,
-    _extract_price_changes,
-    _extract_repair_state,
-    _extract_repair_type,
-    _parse_views,
-    fetch_detail,
-    save_detail_enrichment,
-)
-
-_FIXTURE_DIR = Path(__file__).parent / "fixtures"
-
-# ── _parse_views ──────────────────────────────────────────────────────────────
-
-
-def test_parse_views_space_formatted():
-    assert _parse_views("1 234") == 1234
-
-
-def test_parse_views_plain():
-    assert _parse_views("567") == 567
-
-
-def test_parse_views_none():
-    assert _parse_views(None) is None
-
-
-def test_parse_views_invalid():
-    assert _parse_views("invalid") is None
-
-
-def test_parse_views_comma_formatted():
-    assert _parse_views("1,234") == 1234
-
-
-# ── _extract_repair_state — структурное поле в приоритете, fallback на текст (#622) ──
-
-
-def test_repair_state_structured_wins_over_description():
-    """repairType присутствует → используется он, описание игнорируется."""
-    offer = {
-        "repairType": "cosmetic",  # → standard
-        "description": "Шикарный дизайнерский ремонт",  # инференс дал бы excellent
-    }
-    assert _extract_repair_state(offer) == "standard"
-
-
-def test_repair_state_falls_back_to_description_when_structured_missing():
-    """repairType отсутствует → инференс из описания."""
-    offer = {
-        "repairType": None,
-        "description": "Сделан свежий евроремонт, заезжай и живи",
-    }
-    assert _extract_repair_state(offer) == "good"
-
-
-def test_repair_state_none_when_both_missing():
-    """Нет structured и нет сигнала в тексте → None (не фабрикуем)."""
-    offer = {"description": "Светлая квартира рядом с парком"}
-    assert _extract_repair_state(offer) is None
-
-
-# ── _extract_ceiling_height — offer.building.ceilingHeight, не offer.ceilingHeight (#1791) ──
-
-
-def test_ceiling_height_from_building_node():
-    """На detail-странице высота лежит в offer.building.ceilingHeight."""
-    offer = {"ceilingHeight": None, "building": {"ceilingHeight": "3.0"}}
-    assert _extract_ceiling_height(offer) == 3.0
-
-
-def test_ceiling_height_falls_back_to_top_level():
-    """Если building.ceilingHeight отсутствует — fallback на offer.ceilingHeight."""
-    offer = {"ceilingHeight": "2.75", "building": {}}
-    assert _extract_ceiling_height(offer) == 2.75
-
-
-def test_ceiling_height_none_when_absent_everywhere():
-    assert _extract_ceiling_height({"building": {}}) is None
-    assert _extract_ceiling_height({}) is None
-
-
-# ── _extract_repair_type / state — offer.decoration fallback (#1791) ──────────
-
-
-def test_repair_type_prefers_repair_type_then_decoration():
-    assert _extract_repair_type({"repairType": "euro", "decoration": "cosmetic"}) == "euro"
-    assert _extract_repair_type({"repairType": None, "decoration": "preFine"}) == "preFine"
-    assert _extract_repair_type({}) is None
-
-
-def test_repair_state_from_decoration_when_repair_type_missing():
-    """offer.decoration='preFine' → needs_repair (через нормализатор)."""
-    offer = {"repairType": None, "decoration": "preFine", "description": ""}
-    assert _extract_repair_state(offer) == "needs_repair"
-
-
-# ── _extract_price_changes ────────────────────────────────────────────────────
-
-
-def test_extract_price_changes_normalizes():
-    """curl_cffi flat format: {changeTime, price, diffPercent}."""
-    offer = {
-        "priceChanges": [
-            {"changeTime": "2026-04-01T00:00:00Z", "price": 5000000, "diffPercent": -2.5},
-            {"changeTime": "2026-04-10T00:00:00Z", "price": 4900000, "diffPercent": -2.0},
-        ]
-    }
-    result = _extract_price_changes(offer, {}, {})
-    assert len(result) == 2
-    assert result[0]["change_time"] == "2026-04-01T00:00:00Z"
-    assert result[0]["price_rub"] == 5000000
-    assert result[0]["diff_percent"] == -2.5
-
-
-def test_extract_price_changes_handles_empty_offer():
-    assert _extract_price_changes({}, {}, {}) == []
-
-
-def test_extract_price_changes_handles_none_value():
-    assert _extract_price_changes({"priceChanges": None}, {}, {}) == []
-
-
-def test_extract_price_changes_falls_back_to_state():
-    state = {
-        "priceChanges": [
-            {"changeTime": "2026-03-01T00:00:00Z", "price": 6000000, "diffPercent": 0},
-        ]
-    }
-    result = _extract_price_changes({}, {}, state)
-    assert len(result) == 1
-    assert result[0]["price_rub"] == 6000000
-
-
-def test_extract_price_changes_skips_non_dict_entries():
-    offer = {"priceChanges": [{"changeTime": "2026-04-01T00:00:00Z", "price": 5000000}, "bad"]}
-    result = _extract_price_changes(offer, {}, {})
-    assert len(result) == 1
-
-
-def test_extract_price_changes_offer_data_browser_format():
-    """Browser-rendered HTML: offerData.priceChanges with nested priceData.price."""
-    offer_data = {
-        "priceChanges": [
-            {
-                "changeTime": "2026-06-05T15:32:41Z",
-                "priceData": {"currency": "rur", "price": 13300000},
-            },
-            {
-                "changeTime": "2026-06-03T08:05:04Z",
-                "priceData": {"currency": "rur", "price": 13600000},
-            },
-        ]
-    }
-    result = _extract_price_changes({}, offer_data, {})
-    assert len(result) == 2
-    assert result[0]["change_time"] == "2026-06-05T15:32:41Z"
-    assert result[0]["price_rub"] == 13300000
-    assert result[1]["price_rub"] == 13600000
-
-
-def test_extract_price_changes_offer_wins_over_offer_data():
-    """offer.priceChanges takes priority over offerData.priceChanges (flat format wins)."""
-    offer = {
-        "priceChanges": [
-            {"changeTime": "2026-04-01T00:00:00Z", "price": 5000000},
-        ]
-    }
-    offer_data = {
-        "priceChanges": [
-            {"changeTime": "2026-06-01T00:00:00Z", "priceData": {"price": 9000000}},
-        ]
-    }
-    result = _extract_price_changes(offer, offer_data, {})
-    assert len(result) == 1
-    assert result[0]["price_rub"] == 5000000
-
-
-# ── _extract_agent_profile ────────────────────────────────────────────────────
-
-
-def test_extract_agent_profile_from_agent_key():
-    offer = {
-        "agent": {
-            "id": 12345,
-            "companyName": "Новосёл",
-            "isPro": True,
-            "skills": [{"id": 1, "name": "Продажа"}],
-            "accountType": "specialist",
-        }
-    }
-    profile = _extract_agent_profile(offer)
-    assert profile is not None
-    assert profile["ext_agent_id"] == 12345
-    assert profile["company_name"] == "Новосёл"
-    assert profile["is_pro"] is True
-    assert profile["account_type"] == "specialist"
-
-
-def test_extract_agent_profile_from_user_key():
-    offer = {
-        "user": {
-            "cianUserId": 99999,
-            "companyName": "Агентство Икс",
-            "isPro": False,
-            "accountType": "agency",
-        }
-    }
-    profile = _extract_agent_profile(offer)
-    assert profile is not None
-    assert profile["ext_agent_id"] == 99999
-    assert profile["company_name"] == "Агентство Икс"
-
-
-def test_extract_agent_profile_returns_none_when_no_agent():
-    assert _extract_agent_profile({}) is None
-    assert _extract_agent_profile({"agent": {}, "user": {}}) is None
-
-
-# ── DetailEnrichment defaults ─────────────────────────────────────────────────
-
-
-def test_dataclass_defaults():
-    enrich = DetailEnrichment()
-    assert enrich.cian_id is None
-    assert enrich.price_changes == []
-    assert enrich.raw_sister_states == {}
-    assert enrich.bti_data is None
-    assert enrich.agent_profile is None
-    assert enrich.newbuilding_data is None
-
-
-# ── fetch_detail integration (mocked) ────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_returns_none_on_no_state(monkeypatch):
-    """If state extraction returns None → fetch_detail returns None gracefully."""
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_state",
-        lambda html, mfe, key: None,
-    )
-
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html></html>"
-
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_detail("https://ekb.cian.ru/sale/flat/123/", session=session)
-    assert result is None
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_returns_none_on_non_200(monkeypatch):
-    """Non-200 HTTP response → returns None, no exception raised."""
-    response = MagicMock()
-    response.status_code = 403
-    response.text = ""
-
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_detail("https://ekb.cian.ru/sale/flat/456/", session=session)
-    assert result is None
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_extracts_core_fields(monkeypatch):
-    """Verify core fields are extracted correctly from mocked state."""
-    fake_state = {
-        "offerData": {
-            "offer": {
-                "cianId": 327830237,
-                "windowsViewType": "yardAndStreet",
-                "separateWcsCount": 1,
-                "combinedWcsCount": 0,
-                "ceilingHeight": 2.65,
-                "repairType": "cosmetic",
-                "priceChanges": [
-                    {
-                        "changeTime": "2026-04-01T00:00:00Z",
-                        "price": 5500000,
-                        "diffPercent": -1.8,
-                    }
-                ],
-                "agent": {
-                    "id": 7654321,
-                    "companyName": "Первичка Плюс",
-                    "isPro": True,
-                    "skills": [],
-                    "accountType": "specialist",
-                },
-            },
-            "stats": {
-                "totalViewsFormattedString": "1 042",
-                "todayViewsFormattedString": "7",
-            },
-        }
-    }
-
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_state",
-        lambda html, mfe, key: fake_state if key == "defaultState" else None,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_all_states",
-        lambda html: {},
-    )
-
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html></html>"
-
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_detail("https://ekb.cian.ru/sale/flat/327830237/", session=session)
-    assert result is not None
-    assert result.cian_id == 327830237
-    assert result.windows_view_type == "yardAndStreet"
-    assert result.separate_wcs_count == 1
-    assert result.combined_wcs_count == 0
-    assert result.ceiling_height == 2.65
-    assert result.repair_type == "cosmetic"
-    assert result.views_total == 1042
-    assert result.views_today == 7
-    assert len(result.price_changes) == 1
-    assert result.price_changes[0]["price_rub"] == 5500000
-    assert result.agent_profile is not None
-    assert result.agent_profile["ext_agent_id"] == 7654321
-    assert result.agent_profile["company_name"] == "Первичка Плюс"
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_extracts_bti_from_sister_states(monkeypatch):
-    """BTI sister state is captured into bti_data."""
-    fake_state = {
-        "offerData": {
-            "offer": {
-                "cianId": 111111,
-            }
-        }
-    }
-    fake_all_states = {
-        "frontend-offer-card": {
-            "bti": {
-                "houseData": {
-                    "seriesName": "П-44Т",
-                    "floorsCount": 17,
-                    "flatCount": 120,
-                }
-            }
-        }
-    }
-
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_state",
-        lambda html, mfe, key: fake_state if key == "defaultState" else None,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_all_states",
-        lambda html: fake_all_states,
-    )
-
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html></html>"
-
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_detail("https://ekb.cian.ru/sale/flat/111111/", session=session)
-    assert result is not None
-    assert result.bti_data is not None
-    assert result.bti_data["seriesName"] == "П-44Т"
-    assert "bti" in result.raw_sister_states
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_extracts_newbuilding_link(monkeypatch):
-    """Newbuilding reference is captured from offer.newbuilding."""
-    fake_state = {
-        "offerData": {
-            "offer": {
-                "cianId": 222222,
-                "newbuilding": {
-                    "id": 54016,
-                    "name": "Екатерининский парк",
-                    "url": "https://zhk-ekaterininskiy-park-ekb-i.cian.ru/",
-                },
-            }
-        }
-    }
-
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_state",
-        lambda html, mfe, key: fake_state if key == "defaultState" else None,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_all_states",
-        lambda html: {},
-    )
-
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html></html>"
-
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_detail("https://ekb.cian.ru/sale/flat/222222/", session=session)
-    assert result is not None
-    assert result.newbuilding_data is not None
-    assert result.newbuilding_data["id"] == 54016
-
-
-# ── kitchen_area_m2 extraction ────────────────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_extracts_kitchen_area(monkeypatch):
-    """kitchenArea из offer → kitchen_area_m2 в DetailEnrichment."""
-    fake_state = {
-        "offerData": {
-            "offer": {
-                "cianId": 333333,
-                "kitchenArea": 9.5,
-            }
-        }
-    }
-
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_state",
-        lambda html, mfe, key: fake_state if key == "defaultState" else None,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_all_states",
-        lambda html: {},
-    )
-
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html></html>"
-
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_detail("https://ekb.cian.ru/sale/flat/333333/", session=session)
-    assert result is not None
-    assert result.kitchen_area_m2 == 9.5
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_kitchen_area_none_when_missing(monkeypatch):
-    """Если kitchenArea отсутствует в offer — kitchen_area_m2 остаётся None."""
-    fake_state = {
-        "offerData": {
-            "offer": {
-                "cianId": 444444,
-            }
-        }
-    }
-
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_state",
-        lambda html, mfe, key: fake_state if key == "defaultState" else None,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_all_states",
-        lambda html: {},
-    )
-
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html></html>"
-
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_detail("https://ekb.cian.ru/sale/flat/444444/", session=session)
-    assert result is not None
-    assert result.kitchen_area_m2 is None
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_extracts_description(monkeypatch):
-    """offer.description → DetailEnrichment.description (для persist в listings)."""
-    fake_state = {
-        "offerData": {
-            "offer": {
-                "cianId": 555555,
-                "description": "Продаётся уютная квартира с видом на парк.",
-            }
-        }
-    }
-
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_state",
-        lambda html, mfe, key: fake_state if key == "defaultState" else None,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_all_states",
-        lambda html: {},
-    )
-
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html></html>"
-
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_detail("https://ekb.cian.ru/sale/flat/555555/", session=session)
-    assert result is not None
-    assert result.description == "Продаётся уютная квартира с видом на парк."
-
-
-# ── save_detail_enrichment — kitchen_area_m2 persisted ───────────────────────
-
-
-def test_save_detail_enrichment_writes_kitchen_area():
-    """save_detail_enrichment передаёт kitchen_area_m2 в UPDATE listings."""
-    db = MagicMock()
-
-    # Мокаем fetchone для snapshot-запроса — price_rub=None → snapshot не пишется
-    db.execute.return_value.fetchone.return_value = None
-
-    enrichment = DetailEnrichment(
-        windows_view_type=None,
-        separate_wcs_count=None,
-        combined_wcs_count=None,
-        ceiling_height=None,
-        repair_type=None,
-        repair_state=None,
-        kitchen_area_m2=8.3,
-        views_total=None,
-        views_today=None,
-    )
-
-    save_detail_enrichment(db, listing_id=999, enrichment=enrichment)
-
-    # Проверяем что UPDATE listings вызван с параметром ka=8.3
-    update_call_params = db.execute.call_args_list[0][0][1]
-    assert update_call_params["ka"] == 8.3
-
-
-def test_save_detail_enrichment_kitchen_area_none_passthrough():
-    """Если kitchen_area_m2 = None — параметр ka=None, COALESCE сохраняет существующее."""
-    db = MagicMock()
-    db.execute.return_value.fetchone.return_value = None
-
-    enrichment = DetailEnrichment(kitchen_area_m2=None)
-
-    save_detail_enrichment(db, listing_id=1001, enrichment=enrichment)
-
-    update_call_params = db.execute.call_args_list[0][0][1]
-    assert update_call_params["ka"] is None
-
-
-# ── save_detail_enrichment — description + detail_enriched_at stamp ───────────
-
-
-def test_save_detail_enrichment_writes_description():
-    """save_detail_enrichment передаёт description в UPDATE listings."""
-    db = MagicMock()
-    db.execute.return_value.fetchone.return_value = None
-
-    enrichment = DetailEnrichment(description="Светлая квартира рядом с парком")
-
-    save_detail_enrichment(db, listing_id=1002, enrichment=enrichment)
-
-    update_call_params = db.execute.call_args_list[0][0][1]
-    assert update_call_params["descr"] == "Светлая квартира рядом с парком"
-
-
-def test_save_detail_enrichment_stamps_detail_enriched_at():
-    """UPDATE listings содержит detail_enriched_at = NOW() — иначе строка
-    переобогащается каждый run (detail_enriched_at IS NULL filter)."""
-    db = MagicMock()
-    db.execute.return_value.fetchone.return_value = None
-
-    enrichment = DetailEnrichment(kitchen_area_m2=8.3)
-
-    save_detail_enrichment(db, listing_id=1003, enrichment=enrichment)
-
-    update_sql = str(db.execute.call_args_list[0][0][0])
-    assert "detail_enriched_at = NOW()" in update_sql
-
-
-# ── save_detail_enrichment — diff_percent клампится перед INSERT (#2277) ──────
-# offer_price_history.diff_percent — NUMERIC(8,2) (миграция 131). Этот прямой
-# INSERT не проходит через БД-триггер record_listing_price_change() (тот
-# срабатывает только на UPDATE listings) — garbage/экстремальный diff_percent
-# из скрапленного Cian JSON иначе переполняет колонку.
-
-
-def _insert_diff_param(db: MagicMock) -> float | None:
-    insert_calls = [c for c in db.execute.call_args_list if "offer_price_history" in str(c[0][0])]
-    assert len(insert_calls) == 1
-    return insert_calls[0][0][1]["diff"]
-
-
-def test_save_detail_enrichment_clamps_extreme_positive_diff():
-    db = MagicMock()
-    db.execute.return_value.fetchone.return_value = None
-    enrichment = DetailEnrichment(
-        price_changes=[
-            {"change_time": "2026-04-01T00:00:00Z", "price_rub": 5000000, "diff_percent": 5000000}
-        ]
-    )
-    save_detail_enrichment(db, listing_id=2001, enrichment=enrichment)
-    assert _insert_diff_param(db) == 999999.99
-
-
-def test_save_detail_enrichment_clamps_extreme_negative_diff():
-    db = MagicMock()
-    db.execute.return_value.fetchone.return_value = None
-    enrichment = DetailEnrichment(
-        price_changes=[
-            {
-                "change_time": "2026-04-01T00:00:00Z",
-                "price_rub": 5000000,
-                "diff_percent": -5000000,
-            }
-        ]
-    )
-    save_detail_enrichment(db, listing_id=2002, enrichment=enrichment)
-    assert _insert_diff_param(db) == -999999.99
-
-
-def test_save_detail_enrichment_normal_diff_unchanged():
-    db = MagicMock()
-    db.execute.return_value.fetchone.return_value = None
-    enrichment = DetailEnrichment(
-        price_changes=[
-            {"change_time": "2026-04-01T00:00:00Z", "price_rub": 5000000, "diff_percent": -2.5}
-        ]
-    )
-    save_detail_enrichment(db, listing_id=2003, enrichment=enrichment)
-    assert _insert_diff_param(db) == -2.5
-
-
-def test_save_detail_enrichment_none_diff_unchanged():
-    db = MagicMock()
-    db.execute.return_value.fetchone.return_value = None
-    enrichment = DetailEnrichment(
-        price_changes=[
-            {"change_time": "2026-04-01T00:00:00Z", "price_rub": 5000000, "diff_percent": None}
-        ]
-    )
-    save_detail_enrichment(db, listing_id=2004, enrichment=enrichment)
-    assert _insert_diff_param(db) is None
-
-
-# ── fetch_detail browser_fetcher param ───────────────────────────────────────
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_browser_fetcher_uses_browser_html(monkeypatch):
-    """browser_fetcher provided → HTML comes from BrowserFetcher.fetch, not curl_cffi.
-
-    Simulates the browser-rendered format where priceChanges is at offerData level
-    with nested priceData.price (verified on prod 2026-06-16).
-    """
-    fake_state = {
-        "offerData": {
-            "offer": {
-                "cianId": 328647045,
-            },
-            # Browser-rendered: priceChanges at offerData level, priceData.price format
-            "priceChanges": [
-                {
-                    "changeTime": "2026-06-05T15:32:41Z",
-                    "priceData": {"currency": "rur", "price": 13300000},
-                },
-                {
-                    "changeTime": "2026-06-03T08:05:04Z",
-                    "priceData": {"currency": "rur", "price": 13600000},
-                },
-            ],
-        }
-    }
-
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_state",
-        lambda html, mfe, key: fake_state if key == "defaultState" else None,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_all_states",
-        lambda html: {},
-    )
-
-    # Mock BrowserFetcher: fetch() returns HTML tagged so we can verify it was called.
-    mock_bf = MagicMock()
-    mock_bf.fetch = AsyncMock(return_value="<html>browser-rendered</html>")
-
-    result = await fetch_detail(
-        "https://ekb.cian.ru/sale/flat/328647045/",
-        browser_fetcher=mock_bf,
-    )
-
-    assert result is not None
-    assert result.cian_id == 328647045
-    # price_changes populated from browser-rendered HTML via offerData.priceChanges
-    assert len(result.price_changes) == 2
-    assert result.price_changes[0]["price_rub"] == 13300000
-    assert result.price_changes[0]["change_time"] == "2026-06-05T15:32:41Z"
-    assert result.price_changes[1]["price_rub"] == 13600000
-    # BrowserFetcher.fetch was called with the offer URL
-    mock_bf.fetch.assert_called_once_with("https://ekb.cian.ru/sale/flat/328647045/")
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_browser_fetcher_no_curl_session_used(monkeypatch):
-    """When browser_fetcher is given, curl_cffi AsyncSession is NOT instantiated."""
-    fake_state = {"offerData": {"offer": {"cianId": 555555}}}
-
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_state",
-        lambda html, mfe, key: fake_state if key == "defaultState" else None,
-    )
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_all_states",
-        lambda html: {},
-    )
-
-    # Patch AsyncSession so we can assert it was NOT called.
-    mock_session_cls = MagicMock()
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.AsyncSession",
-        mock_session_cls,
-    )
-
-    mock_bf = MagicMock()
-    mock_bf.fetch = AsyncMock(return_value="<html></html>")
-
-    result = await fetch_detail(
-        "https://ekb.cian.ru/sale/flat/555555/",
-        browser_fetcher=mock_bf,
-    )
-
-    assert result is not None
-    mock_session_cls.assert_not_called()
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_browser_fetcher_returns_none_on_exception(monkeypatch):
-    """If BrowserFetcher.fetch raises, fetch_detail returns None (no crash)."""
-    mock_bf = MagicMock()
-    mock_bf.fetch = AsyncMock(side_effect=RuntimeError("browser service unavailable"))
-
-    result = await fetch_detail(
-        "https://ekb.cian.ru/sale/flat/999999/",
-        browser_fetcher=mock_bf,
-    )
-
-    assert result is None
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_browser_fetcher_state_missing_returns_none(monkeypatch):
-    """browser HTML parsed but no defaultState → fetch_detail returns None."""
-    monkeypatch.setattr(
-        "app.services.scrapers.cian_detail.extract_state",
-        lambda html, mfe, key: None,
-    )
-
-    mock_bf = MagicMock()
-    mock_bf.fetch = AsyncMock(return_value="<html>no state here</html>")
-
-    result = await fetch_detail(
-        "https://ekb.cian.ru/sale/flat/777777/",
-        browser_fetcher=mock_bf,
-    )
-
-    assert result is None
-
-
-# ── Real-card fixture (#1791) — снята с www.cian.ru/sale/flat/330982715/ ──────
-# Регрессия на field-drift: ceiling/kitchen/description должны извлекаться из
-# актуальной structure detail-страницы (defaultState, concat-формат).
-
-
-@pytest.mark.asyncio
-async def test_fetch_detail_real_fixture_extracts_drifted_fields():
-    """На реальной карточке ceiling берётся из offer.building.ceilingHeight,
-    kitchen/description — из offer top-level. До #1791 ceiling был 0%."""
-    html = (_FIXTURE_DIR / "cian_flat_330982715.html").read_text(encoding="utf-8")
-
-    response = MagicMock()
-    response.status_code = 200
-    response.text = html
-
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_detail("https://www.cian.ru/sale/flat/330982715/", session=session)
-
-    assert result is not None
-    # ceiling_height — РАНЬШЕ был None (читали offer.ceilingHeight=null);
-    # теперь из offer.building.ceilingHeight=3.0
-    assert result.ceiling_height == 3.0
-    # kitchen_area_m2 — offer.kitchenArea='14.5' (строка → float)
-    assert result.kitchen_area_m2 == 14.5
-    # description — непустой текст объявления
-    assert result.description is not None
-    assert len(result.description) > 50
-    assert result.cian_id is not None

tradein-mvp/backend/tests/test_cian_serp_scraper.py

@@ -1,792 +0,0 @@
-"""Smoke тесты для cian.py SERP scraper (Stage 3 рефактор).
-
-Проверяют парсинг 137 полей из синтетического Redux state без сетевых запросов.
-Fixture формирует минимальный JS push() conforming to _cianConfig pattern.
-"""
-
-import json
-from unittest.mock import AsyncMock, MagicMock, patch
-
-import pytest
-
-from app.services.scrapers.cian import CianScraper, _format_address
-
-# ── Fixture helpers ──────────────────────────────────────────────────────────
-
-
-def _make_cian_serp_html(offers: list[dict]) -> str:
-    """Обернуть offers в валидный _cianConfig['frontend-serp'].push() HTML."""
-    state = {
-        "results": {
-            "totalOffers": len(offers),
-            "offers": offers,
-        }
-    }
-    state_json = json.dumps(state, ensure_ascii=False)
-    return f"""<html><body>
-<script>
-window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];
-window._cianConfig['frontend-serp'].push({{
-    key: 'initialState',
-    value: {state_json},
-    priority: 10,
-    filter: null
-}});
-</script>
-</body></html>"""
-
-
-def _make_full_offer(
-    cian_id: int = 328739308,
-    price_rur: int = 6500000,
-    rooms: int = 2,
-    total_area: str = "52.0",
-    living_area: str = "35.0",
-    kitchen_area: str = "11.5",
-    floor: int = 7,
-    floors_count: int = 16,
-    year_built: int = 2010,
-    material_type: str = "panel",
-    lat: float = 56.8385,
-    lng: float = 60.5940,
-    cadastral_number: str = "66:41:0204016:1234",
-    building_cadastral_number: str = "66:41:0204016:100",
-    description: str = "Продаётся отличная квартира",
-    description_minhash: str | list[int] = "abc123def456abc1",
-    newbuilding_id: int = 0,
-    has_phones: bool = True,
-    is_homeowner: bool = True,
-    is_pro: bool = False,
-    bargain_allowed: bool = True,
-    sale_type: str = "free",
-) -> dict:
-    """Синтетический Cian offer с полным набором полей (137 полей subset)."""
-    offer: dict = {
-        "cianId": cian_id,
-        "id": cian_id,
-        "fullUrl": f"https://ekb.cian.ru/sale/flat/{cian_id}/",
-        "category": "flatSale",
-        "dealType": "sale",
-        "offerType": "flat",
-        "flatType": "rooms",
-        "isApartments": False,
-        "roomsCount": rooms,
-        "status": "published",
-        # Area
-        "totalArea": total_area,
-        "livingArea": living_area,
-        "kitchenArea": kitchen_area,
-        "balconiesCount": 1,
-        "loggiasCount": 0,
-        "bedroomsCount": None,
-        "hasFurniture": False,
-        "isSoldFurnished": False,
-        "decoration": None,
-        # Floor / building
-        "floorNumber": floor,
-        "building": {
-            "floorsCount": floors_count,
-            "materialType": material_type,
-            "buildYear": year_built,
-            "deadline": None,
-            "parking": None,
-        },
-        # Cadastral
-        "cadastralNumber": cadastral_number,
-        "buildingCadastralNumber": building_cadastral_number,
-        # Price
-        "bargainTerms": {
-            "priceRur": price_rur,
-            "price": price_rur,
-            "currency": "rur",
-            "saleType": sale_type,
-            "mortgageAllowed": True,
-            "bargainAllowed": bargain_allowed,
-            "priceType": "all",
-        },
-        "formattedShortPrice": f"{price_rur // 1000000} млн ₽",
-        # Geo
-        "geo": {
-            "coordinates": {"lat": lat, "lng": lng},
-            "address": [
-                {"type": "location", "name": "Свердловская область"},
-                {"type": "city", "name": "Екатеринбург"},
-                {"type": "street", "name": "улица Постовского"},
-                {"type": "house", "name": "17А"},
-            ],
-            "undergrounds": [
-                {
-                    "name": "Геологическая",
-                    "time": 10,
-                    "transportType": "walk",
-                    "lineId": 10,
-                    "lineColor": "FF0000",
-                    "isDefault": True,
-                    "id": 100,
-                }
-            ],
-            "districts": [],
-        },
-        # Newbuilding
-        "newbuilding": {"id": newbuilding_id, "name": ""},
-        "buildersIds": [],
-        # Seller
-        "phones": [{"number": "9012345678", "countryCode": "7"}] if has_phones else [],
-        "userId": 13332612,
-        "cianUserId": 13332612,
-        "publishedUserId": 13332612,
-        "isByHomeowner": is_homeowner,
-        "isByCommercialOwner": False,
-        "isCianPartner": False,
-        "isPro": is_pro,
-        "isCalltrackingEnabled": False,
-        # Description
-        "description": description,
-        "descriptionMinhash": description_minhash,
-        "isDuplicatedDescription": False,
-        # Time
-        "addedTimestamp": 1716451200,  # 2024-05-23 00:00:00 UTC
-        "added": "вчера",
-        # Photos
-        "photos": [
-            {
-                "id": 1001,
-                "fullUrl": f"https://images.cdn-cian.ru/images/{cian_id}-1.jpg",
-                "thumbnailUrl": f"https://images.cdn-cian.ru/images/{cian_id}-2.jpg",
-            }
-        ],
-        # Verification
-        "isRosreestrChecked": True,
-        "isLayoutApproved": None,
-        # Misc
-        "isNew": False,
-        "isTop3": False,
-        "isPremium": False,
-    }
-    return offer
-
-
-# ── Tests ────────────────────────────────────────────────────────────────────
-
-
-def test_cian_serp_parses_full_offer():
-    """Базовый тест: полный offer → ScrapedLot с ключевыми полями."""
-    offer = _make_full_offer()
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    assert len(lots) == 1
-    lot = lots[0]
-
-    assert lot.source == "cian"
-    assert lot.source_id == "328739308"
-    assert lot.source_url == "https://ekb.cian.ru/sale/flat/328739308/"
-
-
-def test_cian_serp_coords_no_jitter():
-    """Координаты берутся точно из state — NO anchor jitter."""
-    offer = _make_full_offer(lat=56.8385, lng=60.5940)
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    assert len(lots) == 1
-    lot = lots[0]
-    assert lot.lat == pytest.approx(56.8385, abs=1e-9)
-    assert lot.lon == pytest.approx(60.5940, abs=1e-9)
-
-
-def test_cian_serp_kitchen_area_in_raw_payload():
-    """Сырой kitchenArea (str) остаётся в raw_payload (#2008 промоутит float
-    в колонку ScrapedLot.kitchen_area_m2 отдельно — см. соответств. тест)."""
-    offer = _make_full_offer(kitchen_area="11.5")
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.raw_payload is not None
-    assert lot.raw_payload["kitchen_area_m2"] == "11.5"
-
-
-def test_cian_serp_description_minhash():
-    """description_minhash берётся из Cian state, если есть."""
-    offer = _make_full_offer(
-        description_minhash="cian_minhash_abc123",
-        description="Тест описания",
-    )
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.description_minhash == "cian_minhash_abc123"
-
-
-def test_cian_serp_description_minhash_fallback():
-    """Если Cian не дал minhash — вычисляем SHA1 из description."""
-    offer = _make_full_offer(description_minhash="", description="Уникальное описание квартиры")
-    offer["descriptionMinhash"] = ""  # явно пусто
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.description_minhash is not None
-    assert len(lot.description_minhash) == 32  # SHA1 hex [:32]
-
-
-def test_cian_serp_description_minhash_list_int():
-    """descriptionMinhash как list[int] (новый формат Cian ~2026-05-23) → строка через запятую.
-
-    Порядок элементов сохраняется как-есть — sorted() НЕ применяется,
-    т.к. порядок LSH-band'ов значим для dedup/cross-source matching.
-    Offer должен быть принят (не отброшен) — возвращает ScrapedLot, не None.
-    """
-    minhash_list = [8439545, 5781654, 555882, 407717]
-    offer = _make_full_offer(
-        description_minhash=minhash_list,  # type: ignore[arg-type]
-        description="Продаётся квартира с видом на парк",
-    )
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    assert len(lots) == 1, "offer с list[int] minhash должен быть принят, не отброшен"
-    lot = lots[0]
-    assert lot.description_minhash == "8439545,5781654,555882,407717"
-
-
-def test_cian_serp_phones():
-    """PII purge (#2217): даже если raw offer содержит phones, lot.phones пуст —
-    скрейпер больше не читает offer.get("phones")."""
-    offer = _make_full_offer(has_phones=True)
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert isinstance(lot.phones, list)
-    assert lot.phones == []
-
-
-def test_cian_serp_house_ext_id_newbuilding():
-    """Новостройка с кадастром дома → anchor = building cad (#1773), не nb_id.
-
-    _make_full_offer несёт building_cadastral_number, поэтому same-building anchor
-    предпочитает кадастр дома (надёжнее, чем nb_id, который группирует весь ЖК)."""
-    offer = _make_full_offer(newbuilding_id=54016)
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.house_source == "cian_building_cad"
-    assert lot.house_ext_id == "66:41:0204016:100"
-    assert lot.listing_segment == "novostroyki"
-
-
-def test_cian_serp_house_ext_id_newbuilding_nb_id_fallback():
-    """Без кадастра дома anchor падает на nb_id (ЖК) — back-compat fallback."""
-    offer = _make_full_offer(newbuilding_id=54016, building_cadastral_number="")
-    offer["buildingCadastralNumber"] = None
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.house_source == "cian_newbuilding"
-    assert lot.house_ext_id == "54016"
-    assert lot.listing_segment == "novostroyki"
-
-
-def test_cian_serp_vtorichka_no_house_ext_id():
-    """Вторичка (newbuilding.id == 0) — house_ext_id None."""
-    offer = _make_full_offer(newbuilding_id=0)
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.house_source == "cian"
-    assert lot.house_ext_id is None
-    assert lot.listing_segment == "vtorichka"
-
-
-def test_cian_serp_metro_stations():
-    """metro_stations — список dicts с name/time/mode/line_color."""
-    offer = _make_full_offer()
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert isinstance(lot.metro_stations, list)
-    assert len(lot.metro_stations) == 1
-    station = lot.metro_stations[0]
-    assert station["name"] == "Геологическая"
-    assert station["time"] == 10
-    assert station["mode"] == "walk"
-    assert station["line_color"] == "FF0000"
-
-
-def test_cian_serp_cadastral_numbers():
-    """Кадастровые номера квартиры и здания."""
-    offer = _make_full_offer(
-        cadastral_number="66:41:0204016:1234",
-        building_cadastral_number="66:41:0204016:100",
-    )
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.cadastral_number == "66:41:0204016:1234"
-    assert lot.building_cadastral_number == "66:41:0204016:100"
-    # kadastr_num (base field) тоже заполнен
-    assert lot.kadastr_num == "66:41:0204016:1234"
-
-
-def test_cian_serp_is_homeowner_seller():
-    """is_homeowner и is_pro_seller из offer."""
-    offer = _make_full_offer(is_homeowner=True, is_pro=False)
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.is_homeowner is True
-    assert lot.is_pro_seller is False
-
-
-def test_cian_serp_bargain_and_sale_type():
-    """bargain_allowed и sale_type из bargainTerms."""
-    offer = _make_full_offer(bargain_allowed=True, sale_type="alternative")
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.bargain_allowed is True
-    assert lot.sale_type == "alternative"
-
-
-def test_cian_serp_building_fields():
-    """Поля здания: year_built, house_type, total_floors."""
-    offer = _make_full_offer(year_built=1988, material_type="brick", floors_count=9)
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.year_built == 1988
-    assert lot.house_type == "brick"
-    assert lot.total_floors == 9
-
-
-def test_cian_serp_promotes_class_b_fields():
-    """#2008: kitchen_area_m2 / mortgage_available / is_apartments /
-    is_rosreestr_checked промоутятся в колонки ScrapedLot (Class B —
-    дашборд/matching, НЕ valuation)."""
-    # _make_full_offer: mortgageAllowed=True, isApartments=False, isRosreestrChecked=True
-    offer = _make_full_offer(kitchen_area="11.5")
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.kitchen_area_m2 == pytest.approx(11.5)
-    assert lot.mortgage_available is True
-    assert lot.is_apartments is False
-    assert lot.is_rosreestr_checked is True
-
-
-def test_cian_serp_house_type_normalized_monolith_brick():
-    """#2008 (Class A): cian materialType 'monolithBrick' → канон 'monolith_brick';
-    сырьё остаётся в raw_payload['raw_material_type']."""
-    offer = _make_full_offer(material_type="monolithBrick")
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.house_type == "monolith_brick"
-    assert lot.raw_payload is not None
-    assert lot.raw_payload["raw_material_type"] == "monolithBrick"
-
-
-def test_cian_serp_house_type_block_synonyms_normalized():
-    """#2008: gasSilicate/aerocrete/foamConcrete Block + stalin → канон."""
-    scraper = CianScraper()
-    for raw, canon in (
-        ("gasSilicateBlock", "block"),
-        ("aerocreteBlock", "block"),
-        ("foamConcreteBlock", "block"),
-        ("stalin", "brick"),  # «сталинка» = кирпич
-    ):
-        offer = _make_full_offer(material_type=raw)
-        lots = scraper._parse_serp_html(_make_cian_serp_html([offer]))
-        assert lots[0].house_type == canon, f"{raw} → {lots[0].house_type} != {canon}"
-
-
-def test_cian_serp_house_type_canonical_passthrough():
-    """#2008: уже-каноничные monolith/panel остаются неизменными (idempotency)."""
-    scraper = CianScraper()
-    for raw in ("monolith", "panel", "brick", "block", "wood"):
-        offer = _make_full_offer(material_type=raw)
-        lots = scraper._parse_serp_html(_make_cian_serp_html([offer]))
-        assert lots[0].house_type == raw
-
-
-def test_cian_serp_house_type_unknown_becomes_none():
-    """#2008: неизвестный materialType → None (нейтрально для estimator soft-penalty)."""
-    offer = _make_full_offer(material_type="someUnknownMaterial")
-    lots = CianScraper()._parse_serp_html(_make_cian_serp_html([offer]))
-    assert lots[0].house_type is None
-
-
-def test_cian_serp_living_area():
-    """living_area_m2 из offer.livingArea."""
-    offer = _make_full_offer(living_area="35.0")
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.living_area_m2 == pytest.approx(35.0)
-
-
-def test_cian_serp_state_missing_returns_empty():
-    """Если state нет в HTML — возвращаем пустой список."""
-    html = "<html><body><p>no cian state here</p></body></html>"
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-    assert lots == []
-
-
-def test_cian_serp_skips_offer_without_price():
-    """Offer без цены — пропускаем."""
-    offer = _make_full_offer()
-    del offer["bargainTerms"]["priceRur"]
-    del offer["bargainTerms"]["price"]
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-    assert lots == []
-
-
-def test_cian_serp_multiple_offers():
-    """Несколько offers → список ScrapedLot."""
-    offers = [_make_full_offer(cian_id=100 + i) for i in range(5)]
-    html = _make_cian_serp_html(offers)
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-    assert len(lots) == 5
-    ids = [lot.source_id for lot in lots]
-    assert sorted(ids) == ["100", "101", "102", "103", "104"]
-
-
-def test_cian_serp_photo_urls():
-    """photo_urls — полные URL из offer.photos[].fullUrl."""
-    offer = _make_full_offer(cian_id=999)
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert len(lot.photo_urls) == 1
-    assert "cdn-cian.ru" in lot.photo_urls[0]
-    assert "999-1.jpg" in lot.photo_urls[0]
-
-
-def test_cian_serp_no_coords_does_not_crash():
-    """Если координаты отсутствуют — lot.lat/lon = None, не краш."""
-    offer = _make_full_offer()
-    del offer["geo"]["coordinates"]
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    assert len(lots) == 1
-    assert lots[0].lat is None
-    assert lots[0].lon is None
-
-
-# ── Transport: BrowserFetcher (camoufox) миграция (#1806) ────────────────────
-
-
-@pytest.mark.asyncio
-async def test_cian_aenter_opens_browser_fetcher() -> None:
-    """__aenter__ открывает ОДНУ BrowserFetcher(source="cian") сессию (camoufox)."""
-    mock_browser = MagicMock()
-    mock_browser.__aenter__ = AsyncMock(return_value=mock_browser)
-    mock_browser.__aexit__ = AsyncMock(return_value=None)
-
-    captured: dict = {}
-
-    def _fake_browser(*args, **kwargs):
-        captured["source"] = kwargs.get("source") if kwargs else (args[0] if args else None)
-        return mock_browser
-
-    scraper = CianScraper()
-    with patch("app.services.scrapers.cian.BrowserFetcher", _fake_browser):
-        await scraper.__aenter__()
-
-    mock_browser.__aenter__.assert_awaited_once()
-    assert scraper._browser is mock_browser
-    assert captured["source"] == "cian"
-
-    await scraper.__aexit__(None, None, None)
-    mock_browser.__aexit__.assert_awaited_once()
-    assert scraper._browser is None
-
-
-@pytest.mark.asyncio
-async def test_cian_fetch_page_html_uses_browser_fetcher() -> None:
-    """_fetch_page_html тянет HTML через BrowserFetcher.fetch, не через curl_cffi."""
-    scraper = CianScraper.__new__(CianScraper)
-    scraper.name = "cian"
-    scraper.base_url = "https://ekb.cian.ru"
-    scraper.request_delay_sec = 0.0
-
-    fetched_urls: list[str] = []
-
-    async def fake_fetch(url: str) -> str:
-        fetched_urls.append(url)
-        return "<html>serp-page</html>"
-
-    scraper._browser = MagicMock()
-    scraper._browser.fetch = AsyncMock(side_effect=fake_fetch)
-
-    html = await scraper._fetch_page_html((2,), 1, None, None)
-
-    assert html == "<html>serp-page</html>"
-    assert len(fetched_urls) == 1
-    assert "cat.php" in fetched_urls[0]
-
-
-@pytest.mark.asyncio
-async def test_cian_fetch_page_html_returns_none_on_browser_error() -> None:
-    """Если BrowserFetcher.fetch падает — _fetch_page_html возвращает None (не краш)."""
-    scraper = CianScraper.__new__(CianScraper)
-    scraper.name = "cian"
-    scraper.base_url = "https://ekb.cian.ru"
-    scraper.request_delay_sec = 0.0
-
-    scraper._browser = MagicMock()
-    scraper._browser.fetch = AsyncMock(side_effect=RuntimeError("browser down"))
-
-    html = await scraper._fetch_page_html((1,), 2, 1_000_000, 5_000_000)
-    assert html is None
-
-
-@pytest.mark.asyncio
-async def test_cian_fetch_around_parses_browser_html() -> None:
-    """fetch_around парсит лоты из SSR-HTML, отданного BrowserFetcher."""
-    offer = _make_full_offer(cian_id=777)
-    serp_html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper.__new__(CianScraper)
-    scraper.name = "cian"
-    scraper.base_url = "https://ekb.cian.ru"
-    scraper.request_delay_sec = 0.0
-    scraper._browser = MagicMock()
-    scraper._browser.fetch = AsyncMock(return_value=serp_html)
-
-    with patch.object(scraper, "sleep_between_requests", new_callable=AsyncMock):
-        lots = await scraper.fetch_around(56.84, 60.60, rooms=(2,), page=1)
-
-    assert len(lots) == 1
-    assert lots[0].source_id == "777"
-    scraper._browser.fetch.assert_awaited_once()
-
-
-@pytest.mark.asyncio
-async def test_cian_rotate_ip_is_noop() -> None:
-    """_rotate_ip после миграции — no-op, всегда False (прокси ротирует IP сам)."""
-    scraper = CianScraper.__new__(CianScraper)
-    result = await scraper._rotate_ip()
-    assert result is False
-
-
-# ── Address formatter tests ──────────────────────────────────────────────────
-
-
-def test_format_address_standard():
-    """Стандартный geo.address[]."""
-    parts = [
-        {"type": "location", "name": "Свердловская область"},
-        {"type": "city", "name": "Екатеринбург"},
-        {"type": "district", "name": "Чкаловский"},
-        {"type": "street", "name": "улица Постовского"},
-        {"type": "house", "name": "17А"},
-        {"type": "metro", "name": "Геологическая"},
-    ]
-    result = _format_address(parts)
-    # location и metro пропускаем
-    assert "Свердловская" not in result
-    assert "Геологическая" not in result
-    assert "Екатеринбург" in result
-    assert "17А" in result
-
-
-def test_format_address_empty():
-    """Пустой address[] → fallback string."""
-    assert _format_address([]) == "Екатеринбург (Cian)"
-
-
-# ── #1773: новостройка без part{type:"house"} → восстановление номера дома ─────
-
-
-def _make_newbuilding_houseless_offer(
-    cian_id: int = 330047129,
-    newbuilding_id: int = 54016,
-    newbuilding_name: str = "Суходольский квартал",
-    building_cadastral_number: str | None = "66:41:0313010:11396",
-    building_house_number: str | None = None,
-) -> dict:
-    """Cian-оффер новостройки: geo.address[] БЕЗ {type:"house"} (ЖК Суходольский,
-    ул. Евгения Савкова 29, cian 330047129 — реальный кейс из #1773)."""
-    offer = _make_full_offer(
-        cian_id=cian_id,
-        newbuilding_id=newbuilding_id,
-        building_cadastral_number=building_cadastral_number or "",
-    )
-    # geo.address[] без части house — корень бага: «…улица Евгения Савкова» без «29»
-    offer["geo"]["address"] = [
-        {"type": "location", "name": "Свердловская область"},
-        {"type": "city", "name": "Екатеринбург"},
-        {"type": "district", "name": "Академический"},
-        {"type": "street", "name": "улица Евгения Савкова"},
-    ]
-    offer["newbuilding"] = {"id": newbuilding_id, "name": newbuilding_name}
-    building = offer.setdefault("building", {})
-    if building_house_number is not None:
-        building["houseNumber"] = building_house_number
-    if building_cadastral_number is None:
-        offer["buildingCadastralNumber"] = None
-    return offer
-
-
-def test_cian_newbuilding_cad_number_anchor():
-    """Новостройка с кадастром дома → house_source/house_ext_id = building cad
-    (same-building anchor #1773), адрес дополнен кадастром."""
-    offer = _make_newbuilding_houseless_offer(
-        building_cadastral_number="66:41:0313010:11396",
-    )
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    assert len(lots) == 1
-    lot = lots[0]
-    # Кадастр дома предпочтён как anchor-ключ вместо nb_id (ЖК целиком)
-    assert lot.house_source == "cian_building_cad"
-    assert lot.house_ext_id == "66:41:0313010:11396"
-    assert lot.building_cadastral_number == "66:41:0313010:11396"
-    assert lot.listing_segment == "novostroyki"
-    # geo.address[] был без house → кадастр дописан в адрес как якорь
-    assert "Евгения Савкова" in (lot.address or "")
-    assert "66:41:0313010:11396" in (lot.address or "")
-
-
-def test_cian_newbuilding_recovers_house_from_building_number():
-    """building.houseNumber имеет приоритет (a): «29» восстановлен в адрес,
-    кадастр остаётся anchor-ключом."""
-    offer = _make_newbuilding_houseless_offer(
-        building_cadastral_number="66:41:0313010:11396",
-        building_house_number="29",
-    )
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    addr = lot.address or ""
-    assert "Евгения Савкова" in addr
-    assert "29" in addr  # номер дома восстановлен из building.houseNumber
-    # анкор по-прежнему кадастр дома (надёжнее nb_id)
-    assert lot.house_ext_id == "66:41:0313010:11396"
-
-
-def test_cian_newbuilding_recovers_house_from_name_when_no_cad():
-    """Без кадастра дома: дом-токен «29» вытягивается из названия ЖК (приоритет c),
-    anchor падает на nb_id."""
-    offer = _make_newbuilding_houseless_offer(
-        newbuilding_name="ЖК на Савкова 29",
-        building_cadastral_number=None,
-    )
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    assert lot.building_cadastral_number is None
-    # нет кадастра → anchor = nb_id (ЖК)
-    assert lot.house_source == "cian_newbuilding"
-    assert lot.house_ext_id == "54016"
-    addr = lot.address or ""
-    assert "29" in addr  # дом восстановлен из названия ЖК
-
-
-def test_cian_newbuilding_no_house_no_fabrication():
-    """Ни building, ни кадастра, ни номера в названии → номер НЕ выдумываем."""
-    offer = _make_newbuilding_houseless_offer(
-        newbuilding_name="Суходольский квартал",  # без числа-хвоста
-        building_cadastral_number=None,
-    )
-    html = _make_cian_serp_html([offer])
-
-    scraper = CianScraper()
-    lots = scraper._parse_serp_html(html)
-
-    lot = lots[0]
-    addr = lot.address or ""
-    assert "Евгения Савкова" in addr
-    # никаких фейковых catch-all номеров (например 7019) в адресе
-    assert not addr.rstrip().endswith("7019")
-
-
-def test_format_address_vtorichka_houseless_untouched():
-    """Вторичка с house-less адресом НЕ трогается — номер не выдумываем."""
-    parts = [
-        {"type": "city", "name": "Екатеринбург"},
-        {"type": "street", "name": "улица Евгения Савкова"},
-    ]
-    result = _format_address(
-        parts,
-        building={"houseNumber": "29"},
-        building_cadastral_number="66:41:0313010:11396",
-        newbuilding={"id": 0, "name": ""},
-        is_newbuilding=False,
-    )
-    # вторичка: is_newbuilding=False → ни «29», ни кадастр не дописываются
-    assert "29" not in result
-    assert "66:41" not in result

tradein-mvp/backend/tests/test_domclick_sweep.py

@@ -6,21 +6,25 @@ cancel, honest mark_done/mark_failed statuses) удалён вместе с
 теперь в `test_scraper_kit_pipeline_parity2.py::test_domclick_city_sweep_{done,
 blocked_failed}` (kit `run_domclick_city_sweep`).
 
+`app.services.scrapers.domclick` удалён (#2397 Part E2, 0 runtime-импортёров) —
+`_map_item` ретаргетирован на kit `scraper_kit.providers.domclick.serp.DomClickScraper`
+(единственный живой путь).
+
 Остались:
   - DomClickCitySweepCounters (dataclass, из scraper_kit) — defaults + to_dict
-  - DomClickScraper._map_item — BFF offer-item → ScrapedLot маппинг (не зависит
+  - DomClickScraper._map_item (kit) — BFF offer-item → ScrapedLot маппинг (не
-    от оркестрации, `app.services.scrapers.domclick` живёт до Part E2)
+    зависит от оркестрации)
 """
 
 from __future__ import annotations
 
 import os
+from types import SimpleNamespace
 
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test")
 
 import pytest
-
+from scraper_kit.providers.domclick.serp import DomClickScraper
-from app.services.scrapers.domclick import DomClickScraper
 
 # ── DomClickCitySweepCounters ───────────────────────────────────────────────
 
@@ -54,7 +58,8 @@ def test_domclick_sweep_counters_to_dict_all_keys() -> None:
 
 def test_map_item_basic_mapping() -> None:
     """DomClickScraper._map_item маппит BFF offer-item → ScrapedLot корректно."""
-    scraper = DomClickScraper()
+    config = SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
+    scraper = DomClickScraper(config, delay_provider=lambda _name: 8.0)
     item = {
         "id": 111222333,
         "path": "https://domclick.ru/card/sale__flat__111222333",

tradein-mvp/backend/tests/test_houses_rating_filter_entries_1789.py

@@ -1,189 +0,0 @@
-"""#1789 — rating_score из reviews.filterEntries[type==score] (fallback).
-
-Avito перенёс сводный рейтинг из housePage.developmentData.ratingPreview в виджет
-reviews.filterEntries[{type: 'score', value: {scoreFloat, scoreString, reviewCount, ratingStat}}].
-
-Тест 1: когда housePage НЕ даёт рейтинг (ratingBadge пуст), rating берётся из
-        reviews.filterEntries — проверяем rating_score/rating_string/reviews_count/
-        rating_distribution.
-
-Тест 2: housePage даёт рейтинг (старый путь ratingBadge) → filterEntries НЕ
-        перезаписывает (приоритет housePage).
-
-Тест 3: новостройка (developmentPage) без reviews-виджета → graceful, нет краша.
-
-Без сети, без БД.
-"""
-
-from __future__ import annotations
-
-import os
-
-import pytest
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from app.services.scrapers.avito_houses import parse_houses_state
-
-
-def _make_state(
-    *,
-    house_page_rating: dict | None = None,
-    reviews_filter_entries: list | None = None,
-) -> dict:
-    """Минимальный __preloadedState__ с housePage и опциональными reviews.
-
-    house_page_rating — если задан, проставляем в developmentData (старый путь).
-    reviews_filter_entries — если задан, проставляем в reviews.props.reviews.filterEntries
-    """
-    dev_data: dict = {
-        "avitoId": 555666,
-        "id": "aGFzaA==",
-        "title": "Test House",
-        "address": "ул. Тестовая, 1",
-        "coords": {"lat": 56.8, "lng": 60.6},
-        "aboutDevelopment": {"expandParams": {"items": []}},
-        "developer": {},
-        "mapPreview": {},
-    }
-    if house_page_rating is not None:
-        dev_data.update(house_page_rating)
-
-    placeholders: list = [
-        {
-            "type": "housePage",
-            "props": {"developmentData": dev_data},
-        },
-    ]
-
-    if reviews_filter_entries is not None:
-        placeholders.append(
-            {
-                "type": "reviews",
-                "props": {
-                    # MFE-форма: данные под ключом "reviews" внутри props
-                    "reviews": {
-                        "entries": [],
-                        "filterEntries": reviews_filter_entries,
-                    }
-                },
-            }
-        )
-
-    return {"data": {"data": {"page": {"placeholders": placeholders}}}}
-
-
-# ---------------------------------------------------------------------------
-# Тест 1: рейтинг из reviews.filterEntries (housePage пуст)
-# ---------------------------------------------------------------------------
-
-
-def test_rating_from_filter_entries_when_housepage_empty() -> None:
-    """Когда housePage не даёт рейтинг — берём из reviews.filterEntries."""
-    state = _make_state(
-        reviews_filter_entries=[
-            {
-                "type": "score",
-                "value": {
-                    "scoreFloat": 4.666,
-                    "scoreString": "4,7",
-                    "reviewCount": 3,
-                    "ratingStat": [
-                        {"score": 5, "count": 2},
-                        {"score": 4, "count": 1},
-                    ],
-                },
-            }
-        ]
-    )
-
-    e = parse_houses_state(state, "/catalog/houses/ekaterinburg/test/555666")
-    h = e.house
-
-    assert h.rating_score == pytest.approx(4.666, rel=1e-3)
-    assert h.rating_string == "4,7"
-    assert h.reviews_count == 3
-    assert len(h.rating_distribution) == 2
-    assert h.rating_distribution[0]["score"] == 5
-
-
-def test_rating_filter_entries_float_type() -> None:
-    """scoreFloat из filterEntries конвертируется в float (не остаётся int/str)."""
-    state = _make_state(
-        reviews_filter_entries=[
-            {
-                "type": "score",
-                "value": {"scoreFloat": 5, "scoreString": "5,0", "reviewCount": 1},
-            }
-        ]
-    )
-    e = parse_houses_state(state, "/catalog/houses/ekaterinburg/test/555666")
-    assert isinstance(e.house.rating_score, float)
-    assert e.house.rating_score == 5.0
-
-
-# ---------------------------------------------------------------------------
-# Тест 2: housePage даёт рейтинг → filterEntries НЕ перезаписывает
-# ---------------------------------------------------------------------------
-
-
-def test_housepage_rating_takes_priority_over_filter_entries() -> None:
-    """Если housePage.ratingBadge уже заполнен — filterEntries не перезаписывает."""
-    state = _make_state(
-        house_page_rating={
-            "ratingBadge": {"info": {"score": 3.9, "scoreString": "3,9"}},
-            "ratingSummaryStat": {"reviewCount": 10, "ratingStat": []},
-        },
-        reviews_filter_entries=[
-            {
-                "type": "score",
-                "value": {
-                    "scoreFloat": 4.9,
-                    "scoreString": "4,9",
-                    "reviewCount": 99,
-                    "ratingStat": [],
-                },
-            }
-        ],
-    )
-
-    e = parse_houses_state(state, "/catalog/houses/ekaterinburg/test/555666")
-    h = e.house
-
-    # Значение из housePage должно сохраниться
-    assert h.rating_score == pytest.approx(3.9, rel=1e-3)
-    assert h.rating_string == "3,9"
-    assert h.reviews_count == 10
-
-
-# ---------------------------------------------------------------------------
-# Тест 3: нет reviews-виджета вообще → None, не падаем
-# ---------------------------------------------------------------------------
-
-
-def test_no_reviews_widget_rating_is_none() -> None:
-    """Без виджета reviews и без ratingBadge → rating_score=None, graceful."""
-    state = _make_state()  # нет reviews_filter_entries
-
-    e = parse_houses_state(state, "/catalog/houses/ekaterinburg/test/555666")
-    h = e.house
-
-    assert h.rating_score is None
-    assert h.rating_string is None
-    assert h.reviews_count is None
-
-
-# ---------------------------------------------------------------------------
-# Тест 4: filterEntries без entry type==score → rating остаётся None
-# ---------------------------------------------------------------------------
-
-
-def test_filter_entries_without_score_type() -> None:
-    """filterEntries без type==score → рейтинг не заполняется."""
-    state = _make_state(
-        reviews_filter_entries=[
-            {"type": "something_else", "value": {"scoreFloat": 9.9}},
-        ]
-    )
-    e = parse_houses_state(state, "/catalog/houses/ekaterinburg/test/555666")
-    assert e.house.rating_score is None

tradein-mvp/backend/tests/test_pipeline_browser_routing.py

@@ -12,6 +12,10 @@ Legacy `run_avito_pipeline` browser/curl-mode integration tests (было ран
 удалены вместе с `app.services.scrape_pipeline` (#2397 Part E1) — browser-routing
 покрытие на уровне `fetch_detail`/`fetch_house_catalog`/`AvitoScraper._fetch_serp_html`
 ниже остаётся нетронутым.
+
+`fetch_house_catalog` ретаргетирован на `scraper_kit.providers.avito.houses` (#2397
+Part E2) — legacy `app.services.scrapers.avito_houses` удалён (0 runtime-импортёров,
+kit — единственный живой путь).
 """
 
 from __future__ import annotations
@@ -26,9 +30,11 @@ import pytest
 
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
 
+from scraper_kit.avito_exceptions import AvitoBlockedError as KitAvitoBlockedError
+from scraper_kit.providers.avito.houses import fetch_house_catalog
+
 from app.services.scrapers.avito_detail import fetch_detail
 from app.services.scrapers.avito_exceptions import AvitoBlockedError
-from app.services.scrapers.avito_houses import fetch_house_catalog
 
 FIXTURES = Path(__file__).parent / "fixtures"
 
@@ -205,10 +211,10 @@ async def test_fetch_house_catalog_browser_no_curl_used() -> None:
 
 @pytest.mark.asyncio
 async def test_fetch_house_catalog_browser_firewall_raises() -> None:
-    """Firewall HTML через browser path → AvitoBlockedError."""
+    """Firewall HTML через browser path → AvitoBlockedError (kit)."""
     mock_fetcher = make_mock_fetcher(FIREWALL_HTML)
 
-    with pytest.raises(AvitoBlockedError, match="browser-mode"):
+    with pytest.raises(KitAvitoBlockedError, match="browser-mode"):
         await fetch_house_catalog(
             "/catalog/houses/ekaterinburg/ul_test/9999",
             browser_fetcher=mock_fetcher,

tradein-mvp/backend/tests/test_scraper_kit_bisection_parity.py

@@ -1,14 +1,15 @@
-"""Golden-parity бисекции: старый боевой обход `app.services.scrapers.<provider>`
+"""Golden-parity бисекции: старый боевой обход `app.services.scrapers.avito`
 ≡ новый единый движок `scraper_kit.pricing.walk_price_range` (#2134).
 
 Четыре независимые копии адаптивной бисекции «комнатность × цена» были
 консолидированы в один движок (`scraper_kit/pricing/bisection.py`). Провайдеры
-(avito/cian/yandex/domclick) теперь передают в движок свои probe/paginate + свой
+cian/domclick/yandex_realty удалены (#2397 Part E2, 0 runtime-импортёров) —
-`BisectionConfig`. Этот тест доказывает, что unification НЕ изменил ОБХОД: для
+их legacy-vs-kit секции обхода убраны отсюда вместе с ними (легаси-эталон
-каждого провайдера при одинаковом входе (детерминированный мок счётчика по
+исчез). `avito` остаётся легаси (не мигрирован) — его parity-проверка ниже
-диапазону) старая боевая реализация и новая (движок) посещают ИДЕНТИЧНУЮ
+доказывает, что unification НЕ изменил ОБХОД: при одинаковом входе
-последовательность бакетов — probe-запросы, leaf-пагинации, деления, ранние
+(детерминированный мок счётчика по диапазону) старая боевая реализация и
-возвраты на нуле, probe-fail-ветки.
+новая (движок) посещают ИДЕНТИЧНУЮ последовательность бакетов — probe-запросы,
+leaf-пагинации, деления, ранние возвраты на нуле, probe-fail-ветки.
 
 Механика: на ОБЕ реализации (старую и новую) ставятся ОДИНАКОВЫЕ моки на
 низкоуровневые методы (probe-fetch + leaf-пагинатор), которые (а) возвращают
@@ -22,7 +23,7 @@ import os
 from collections.abc import Callable
 from types import SimpleNamespace
 from typing import Any
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import AsyncMock, patch
 
 import pytest
 
@@ -30,14 +31,8 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/
 
 from scraper_kit.price_brackets import get_price_seed_brackets
 from scraper_kit.providers.avito.serp import AvitoScraper as NewAvito
-from scraper_kit.providers.cian.serp import CianScraper as NewCian
-from scraper_kit.providers.domclick.serp import DomClickScraper as NewDomClick
-from scraper_kit.providers.yandex.serp import YandexRealtyScraper as NewYandex
 
 from app.services.scrapers.avito import AvitoScraper as OldAvito
-from app.services.scrapers.cian import CianScraper as OldCian
-from app.services.scrapers.domclick import DomClickScraper as OldDomClick
-from app.services.scrapers.yandex_realty import YandexRealtyScraper as OldYandex
 
 # ── Оракул счётчиков ─────────────────────────────────────────────────────────
 
@@ -55,7 +50,7 @@ def make_oracle(
 
     span считается по клампнутым границам (None-lo → 0, None-hi → open_ceiling),
     чтобы широкие брекеты делились, а узкие уходили в leaf. `zeros` форсит 0 (ранний
-    возврат / leaf-на-нуле у yandex), `nones` форсит None (probe-fail).
+    возврат), `nones` форсит None (probe-fail).
     """
 
     def oracle(lo: Any, hi: Any) -> int | None:
@@ -71,10 +66,10 @@ def make_oracle(
     return oracle
 
 
-# ── Провайдер-специфичные «сборщики событий» ─────────────────────────────────
+# ── avito «сборщик событий» ──────────────────────────────────────────────────
 #
-# Каждый ставит одинаковые моки на старый/новый экземпляр и возвращает список
+# Ставит одинаковые моки на старый/новый экземпляр и возвращает список событий
-# событий обхода. `("probe", lo, hi)` — запрос счётчика бакета; `("leaf", ...)` —
+# обхода. `("probe", lo, hi)` — запрос счётчика бакета; `("leaf", ...)` —
 # пагинация leaf-бакета.
 
 
@@ -91,81 +86,6 @@ def _patch_avito(scraper: Any, oracle: Oracle, events: list[Any]) -> None:
     scraper._rotate_ip = AsyncMock(return_value=False)
 
 
-def _patch_cian(scraper: Any, oracle: Oracle, events: list[Any]) -> None:
-    holder: dict[str, tuple[Any, Any]] = {}
-
-    def fph(rooms: Any, page: int, min_price: Any, max_price: Any) -> str:
-        events.append(("probe", min_price, max_price))
-        holder["b"] = (min_price, max_price)
-        return "H"
-
-    def eto(html: str) -> int | None:
-        return oracle(*holder["b"])
-
-    def leaf(**kw: Any) -> None:
-        events.append(("leaf", kw["lo"], kw["hi"], kw["total"]))
-
-    scraper._fetch_page_html = AsyncMock(side_effect=fph)
-    scraper._extract_total_offers = MagicMock(side_effect=eto)
-    scraper._paginate_leaf_bucket = AsyncMock(side_effect=leaf)
-    scraper._rotate_ip = AsyncMock(return_value=False)
-    scraper.sleep_between_requests = AsyncMock()
-
-
-def _patch_domclick(scraper: Any, oracle: Oracle, events: list[Any]) -> None:
-    def count(fetcher: Any, rooms: Any, gte: Any, lte: Any) -> int:
-        events.append(("probe", gte, lte))
-        val = oracle(gte, lte)
-        return val if val is not None else 0
-
-    def paginate(
-        fetcher: Any, rooms: Any, gte: Any, lte: Any, seen_ids: Any, out_lots: Any, pages: Any
-    ) -> None:
-        events.append(("leaf", gte, lte))
-
-    scraper._count = AsyncMock(side_effect=count)
-    scraper._paginate = AsyncMock(side_effect=paginate)
-
-
-def _patch_yandex(
-    scraper: Any, oracle: Oracle, events: list[Any], *, record_pages: bool = False
-) -> None:
-    """Yandex: leaf-пагинация встроена в walk (нет отдельного метода-хука).
-
-    Стабим `_fetch_page_json` (page=1 отдаёт payload с totalItems из оракула;
-    page>1 — пустой payload) и передаём `on_bucket`, который пишет завершение
-    leaf/degraded бакета. probe = page-1-fetch; leaf-страницы 2..N идут через
-    asyncio.gather (порядок недетерминирован) — их НЕ пишем в норм-режиме; факт
-    leaf-обхода фиксируется через on_bucket. Для degraded-теста (пагинация
-    последовательная) `record_pages=True` пишет все страницы.
-    """
-
-    def gate(total: int | None, entities: list[Any] | None = None) -> dict[str, Any]:
-        pager: dict[str, Any] = {}
-        if total is not None:
-            pager["totalItems"] = total
-        return {"response": {"search": {"offers": {"entities": entities or [], "pager": pager}}}}
-
-    def fpj(rooms: Any, page: int, price_min: Any, price_max: Any, new_flat: str = "NO") -> Any:
-        if page == 1:
-            events.append(("probe", price_min, price_max))
-            return gate(oracle(price_min, price_max))
-        if record_pages:
-            events.append(("page", price_min, price_max, page))
-        return gate(0, entities=[])
-
-    scraper._fetch_page_json = AsyncMock(side_effect=fpj)
-    scraper._rotate_ip = AsyncMock(return_value=False)
-    scraper.request_delay_sec = 0.0
-
-
-def _yandex_on_bucket(events: list[Any]) -> Callable[[str, int], None]:
-    def on_bucket(bucket_key: str, count: int) -> None:
-        events.append(("bucket", bucket_key))
-
-    return on_bucket
-
-
 # ── Конструкторы старых/новых экземпляров (offline, без DB) ───────────────────
 
 _KIT_CONFIG = SimpleNamespace(
@@ -185,42 +105,14 @@ def _old_avito() -> Any:
         return OldAvito()
 
 
-def _old_cian() -> Any:
+# ── avito: probe → split → leaf ───────────────────────────────────────────────
-    with patch("app.services.scrapers.cian.get_scraper_delay", return_value=5.0):
-        return OldCian()
 
-
-def _old_yandex() -> Any:
-    with patch("app.services.scrapers.yandex_realty.get_scraper_delay", return_value=5.0):
-        return OldYandex()
-
-
-def _old_domclick() -> Any:
-    with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=8.0):
-        return OldDomClick()
-
-
-# ── avito / cian: одинаковый скелет (probe → split → leaf) ────────────────────
-
-
-# Провайдер-специфичные kwargs для _walk_price_range (avito идентифицирует бакет
-# room_slug/room_label; cian — кортежем rooms).
 _AVITO_WALK_EXTRA = {"room_slug": "room1", "room_label": "1к"}
-_CIAN_WALK_EXTRA = {"rooms": (1,)}
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize(
+async def test_bisection_parity_avito() -> None:
-    "old_factory, new_factory, patch_fn, walk_extra",
+    """avito: split-дерево + leaf + ранний возврат на нуле — обход идентичен.
-    [
-        (_old_avito, lambda: NewAvito(config=_KIT_CONFIG), _patch_avito, _AVITO_WALK_EXTRA),
-        (_old_cian, lambda: NewCian(config=_KIT_CONFIG), _patch_cian, _CIAN_WALK_EXTRA),
-    ],
-)
-async def test_bisection_parity_avito_cian(
-    old_factory: Any, new_factory: Any, patch_fn: Any, walk_extra: dict[str, Any]
-) -> None:
-    """avito/cian: split-дерево + leaf + ранний возврат на нуле — обход идентичен.
 
     Оракул: divisor=1000, cap=1400 → брекет 8М делится 3 уровня (8М→4М→2М→1М-leaf);
     один под-брекет форсим в 0 (ранний возврат без leaf).
@@ -237,7 +129,7 @@ async def test_bisection_parity_avito_cian(
 
     async def run(scraper: Any) -> list[Any]:
         events: list[Any] = []
-        patch_fn(scraper, oracle, events)
+        _patch_avito(scraper, oracle, events)
         for lo, hi in brackets:
             await scraper._walk_price_range(
                 lo=lo,
@@ -249,12 +141,12 @@ async def test_bisection_parity_avito_cian(
                 secondary_only=True,
                 on_bucket=None,
                 skip_buckets=None,
-                **walk_extra,
+                **_AVITO_WALK_EXTRA,
             )
         return events
 
-    old_events = await run(old_factory())
+    old_events = await run(_old_avito())
-    new_events = await run(new_factory())
+    new_events = await run(NewAvito(config=_KIT_CONFIG))
     assert old_events == new_events
     # Санити: обход реально нетривиален (несколько probe + несколько leaf).
     assert sum(1 for e in old_events if e[0] == "leaf") >= 4
@@ -300,140 +192,18 @@ async def test_bisection_parity_avito_probe_fail() -> None:
     assert ("leaf", 25_000_000, None, 100) in old_events
 
 
-# ── domclick: count-endpoint + None-both-ends + split открытого корня ─────────
-
-
-@pytest.mark.asyncio
-async def test_bisection_parity_domclick() -> None:
-    """domclick: unbounded-корень (None,None) клампится к [0,1e9] и делится;
-    count-endpoint; span<=MIN_PRICE_SPAN строго → leaf. Обход идентичен старому."""
-    oracle = make_oracle(divisor=100_000, open_ceiling=1_000_000_000)
-
-    async def run(scraper: Any) -> list[Any]:
-        events: list[Any] = []
-        _patch_domclick(scraper, oracle, events)
-        await scraper._sweep_bucket(
-            fetcher=None,
-            rooms="1",
-            price_gte=None,
-            price_lte=None,
-            seen_ids=set(),
-            out_lots=[],
-            pages=100,
-        )
-        return events
-
-    old_events = await run(_old_domclick())
-    new_events = await run(NewDomClick(_make_domclick_config()))
-    assert old_events == new_events
-    assert sum(1 for e in old_events if e[0] == "leaf") >= 4
-    # Корень (None,None) обязательно пробится первым.
-    assert old_events[0] == ("probe", None, None)
-
-
-def _make_domclick_config() -> Any:
-    return SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
-
-
-# ── yandex: max_depth=8, min_bracket=500k, count-based on_bucket ──────────────
-
-
-@pytest.mark.asyncio
-async def test_bisection_parity_yandex_normal() -> None:
-    """yandex: split-дерево (порог 500k, cap 500) + leaf-on-bucket + zero-as-leaf.
-
-    Оракул divisor=1000, cap=500 → 4М→2М→1М→500k-leaf. Один под-брекет=0
-    (yandex вызывает on_bucket даже на нуле — treat_zero_as_leaf).
-    """
-    oracle = make_oracle(
-        divisor=1000,
-        open_ceiling=30_000_000,
-        zeros=frozenset({(2_000_000, 2_999_999)}),
-    )
-    brackets: list[tuple[int, int | None]] = [(0, 3_999_999), (25_000_000, None)]
-
-    async def run(scraper: Any) -> list[Any]:
-        events: list[Any] = []
-        _patch_yandex(scraper, oracle, events)
-        for lo, hi in brackets:
-            await scraper._walk_price_range(
-                rooms="1",
-                lo=lo,
-                hi=hi,
-                seen={},
-                price_cap_per_bucket=500,
-                max_pages_per_bucket=50,
-                concurrency=4,
-                on_bucket=_yandex_on_bucket(events),
-                skip_buckets=None,
-            )
-        return events
-
-    old_events = await run(_old_yandex())
-    new_events = await run(NewYandex(config=_KIT_CONFIG))
-    assert old_events == new_events
-    assert sum(1 for e in old_events if e[0] == "bucket") >= 4
-    assert sum(1 for e in old_events if e[0] == "probe") >= 6
-
-
-@pytest.mark.asyncio
-async def test_bisection_parity_yandex_degraded() -> None:
-    """yandex probe-fail (totalItems неизвестен) → degraded paginate-until-empty.
-
-    Оракул форсит None на бакете → probe page1 payload без totalItems → total=None
-    → degraded последовательно пагинирует. record_pages=True фиксирует все страницы.
-    """
-    oracle = make_oracle(
-        divisor=1000,
-        open_ceiling=30_000_000,
-        nones=frozenset({(None, 3_999_999)}),
-    )
-
-    async def run(scraper: Any) -> list[Any]:
-        events: list[Any] = []
-        _patch_yandex(scraper, oracle, events, record_pages=True)
-        await scraper._walk_price_range(
-            rooms="1",
-            lo=0,
-            hi=3_999_999,
-            seen={},
-            price_cap_per_bucket=500,
-            max_pages_per_bucket=50,
-            concurrency=4,
-            on_bucket=_yandex_on_bucket(events),
-            skip_buckets=None,
-        )
-        return events
-
-    old_events = await run(_old_yandex())
-    new_events = await run(NewYandex(config=_KIT_CONFIG))
-    assert old_events == new_events
-    # degraded: probe page1 + затем последовательная пагинация страниц 1..K до пустой.
-    assert ("probe", None, 3_999_999) in old_events
-    assert any(e[0] == "bucket" for e in old_events)
-
-
 # ── Full seed-grid обход (все seed-брекеты одной комнатности) ─────────────────
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize(
+async def test_bisection_parity_full_seed_grid() -> None:
-    "old_factory, new_factory, patch_fn, walk_extra",
-    [
-        (_old_avito, lambda: NewAvito(config=_KIT_CONFIG), _patch_avito, _AVITO_WALK_EXTRA),
-        (_old_cian, lambda: NewCian(config=_KIT_CONFIG), _patch_cian, _CIAN_WALK_EXTRA),
-    ],
-)
-async def test_bisection_parity_full_seed_grid(
-    old_factory: Any, new_factory: Any, patch_fn: Any, walk_extra: dict[str, Any]
-) -> None:
     """Полный обход всех seed-брекетов (как в fetch_all_secondary) — включая
-    открытый catch-all — идентичен между старым и новым для avito/cian."""
+    открытый catch-all — идентичен между старым и новым для avito."""
     oracle = make_oracle(divisor=1500, open_ceiling=300_000_000)
 
     async def run(scraper: Any) -> list[Any]:
         events: list[Any] = []
-        patch_fn(scraper, oracle, events)
+        _patch_avito(scraper, oracle, events)
         for br_lo, br_hi in get_price_seed_brackets():
             walk_hi = br_hi - 1 if br_hi is not None else None
             await scraper._walk_price_range(
@@ -446,10 +216,10 @@ async def test_bisection_parity_full_seed_grid(
                 secondary_only=True,
                 on_bucket=None,
                 skip_buckets=None,
-                **walk_extra,
+                **_AVITO_WALK_EXTRA,
             )
         return events
 
-    old_events = await run(old_factory())
+    old_events = await run(_old_avito())
-    new_events = await run(new_factory())
+    new_events = await run(NewAvito(config=_KIT_CONFIG))
     assert old_events == new_events

tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py

@@ -1,20 +1,22 @@
-"""Golden-parity: `scraper_kit.providers.cian.*` парсинг ≡ `app.services.scrapers.cian*`.
+"""Golden-parity: `scraper_kit.providers.cian.*` парсинг ≡ ещё живым `app.services.scrapers.cian*`.
 
 Strangler-инвариант (#2133 cian): новая scraper_kit-копия cian-провайдера должна давать
 БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. Развязка
 (settings→ScraperConfig, get_scraper_delay→delay_provider, cian_session→локальная копия)
 НЕ меняет распарсенные данные.
 
-Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
+`app.services.scrapers.cian` / `cian_detail` удалены (#2397 Part E2, 0 runtime-
-  - SERP `_parse_serp_html` (Redux state → list[ScrapedLot]) на синтетическом _cianConfig
+импортёров — admin.py/cian_history_backfill.py уже на kit). Parity-тесты SERP
-    push-HTML (вторичка + новостройка + студия), сравнение model_dump();
+`_parse_serp_html`/`_offer_to_lot`/`_extract_total_offers`/`_format_address` и
-  - SERP `_offer_to_lot` пооферно + `_format_address` (pure, восстановление дома ЖК);
+detail `fetch_detail` удалены вместе с ними (легаси-эталон исчез).
-  - `_extract_total_offers` (totalOffers из state);
+
-  - `extract_state` — app-версия ≡ kit-версия на одном HTML (общий парсер state);
+Оставшееся здесь сравнивает kit ≡ ЕЩЁ ЖИВЫЕ легаси-модули:
-  - detail `fetch_detail` (defaultState + bti sister → DetailEnrichment, dataclasses.asdict);
+  - `extract_state` — `app.services.scrapers.cian_state_parser` (живой, общий парсер state);
-  - valuation `_parse_valuation_state` (estimation/chart/houseInfo → CianValuationResult);
+  - valuation `_parse_valuation_state` / `estimate_via_cian_valuation` —
-  - newbuilding pure-экстракторы (_extract_chart / _extract_reliability_checks /
+    `app.services.scrapers.cian_valuation` (живой);
-    _extract_nested_offers / _extract_zhk_url_from_serp).
+  - newbuilding pure-экстракторы — `app.services.scrapers.cian_newbuilding` (живой);
+  - kit mandatory-config guards (`estimate_via_cian_valuation`, `cian.session`) — kit-only;
+  - strangler-guard: kit-провайдер не импортирует `app.*`.
 
 Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
 """
@@ -27,17 +29,17 @@ import json
 import os
 from types import SimpleNamespace
 from typing import Any
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import MagicMock
 
 import pytest
 
-# Старый app.services.scrapers.cian* импортирует app.core.config.settings=Settings(),
+# Старый app.services.scrapers.cian_state_parser/cian_valuation/cian_newbuilding
-# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
+# импортируют app.core.config.settings=Settings(), которому нужен DATABASE_URL.
+# Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
 
 # НОВЫЙ (scraper_kit) cian-провайдер.
 from scraper_kit.cian_state_parser import extract_state as new_extract_state
-from scraper_kit.providers.cian.detail import fetch_detail as new_fetch_detail
 from scraper_kit.providers.cian.newbuilding import _extract_chart as new_extract_chart
 from scraper_kit.providers.cian.newbuilding import (
     _extract_nested_offers as new_extract_nested_offers,
@@ -48,8 +50,6 @@ from scraper_kit.providers.cian.newbuilding import (
 from scraper_kit.providers.cian.newbuilding import (
     _extract_zhk_url_from_serp as new_extract_zhk_url,
 )
-from scraper_kit.providers.cian.serp import CianScraper as NewCianScraper
-from scraper_kit.providers.cian.serp import _format_address as new_format_address
 from scraper_kit.providers.cian.valuation import (
     _parse_valuation_state as new_parse_valuation,
 )
@@ -62,10 +62,7 @@ from scraper_kit.providers.cian.valuation import (
 # #2306's cian_price_history.py (см. app/services/scraper_adapters.py).
 from app.services.scraper_adapters import RealScraperConfig
 
-# СТАРЫЙ (app.services.scrapers) cian-провайдер.
+# СТАРЫЙ (app.services.scrapers) — ещё живые cian-модули (cian.py/cian_detail.py удалены).
-from app.services.scrapers.cian import CianScraper as OldCianScraper
-from app.services.scrapers.cian import _format_address as old_format_address
-from app.services.scrapers.cian_detail import fetch_detail as old_fetch_detail
 from app.services.scrapers.cian_newbuilding import _extract_chart as old_extract_chart
 from app.services.scrapers.cian_newbuilding import (
     _extract_nested_offers as old_extract_nested_offers,
@@ -85,14 +82,11 @@ from app.services.scrapers.cian_valuation import (
 )
 from tests.support.parity import assert_parity
 
-# ── Fixtures: SERP offers → _cianConfig push HTML ────────────────────────────
+# ── extract_state parity (state-парсер, живой в cian_state_parser.py) ────────
-
-_CONFIG = SimpleNamespace(glitchtip_dsn=None)
 
 
-def _serp_html(offers: list[dict[str, Any]]) -> str:
+def _serp_html_for_state(state: dict[str, Any]) -> str:
-    """Обернуть offers в валидный _cianConfig['frontend-serp'].push() HTML."""
+    """Обернуть произвольный state в валидный _cianConfig['frontend-serp'].push() HTML."""
-    state = {"results": {"totalOffers": len(offers), "offers": offers}}
     state_json = json.dumps(state, ensure_ascii=False)
     return (
         "<html><body><script>\n"
@@ -103,293 +97,16 @@ def _serp_html(offers: list[dict[str, Any]]) -> str:
     )
 
 
-def _vtorichka_offer() -> dict[str, Any]:
-    return {
-        "cianId": 328739308,
-        "id": 328739308,
-        "fullUrl": "https://ekb.cian.ru/sale/flat/328739308/",
-        "offerType": "flat",
-        "flatType": "rooms",
-        "category": "flatSale",
-        "isApartments": False,
-        "roomsCount": 2,
-        "totalArea": "52.0",
-        "livingArea": "35.0",
-        "kitchenArea": "11.5",
-        "balconiesCount": 1,
-        "loggiasCount": 0,
-        "bedroomsCount": None,
-        "hasFurniture": False,
-        "isSoldFurnished": False,
-        "decoration": "euro",
-        "floorNumber": 7,
-        "building": {
-            "floorsCount": 16,
-            "materialType": "monolithBrick",
-            "buildYear": 2010,
-            "parking": {"type": "underground"},
-            "totalArea": "3400",
-        },
-        "cadastralNumber": "66:41:0204016:1234",
-        "buildingCadastralNumber": "66:41:0204016:100",
-        "bargainTerms": {
-            "priceRur": 6500000,
-            "price": 6500000,
-            "saleType": "free",
-            "mortgageAllowed": True,
-            "bargainAllowed": True,
-        },
-        "geo": {
-            "coordinates": {"lat": 56.8385, "lng": 60.5940},
-            "address": [
-                {"type": "location", "fullName": "Свердловская область"},
-                {"type": "raion", "fullName": "Кировский"},
-                {"type": "street", "fullName": "улица Постовского"},
-                {"type": "house", "fullName": "17А"},
-                {"type": "metro", "fullName": "Геологическая"},
-            ],
-            "undergrounds": [
-                {
-                    "name": "Геологическая",
-                    "time": 10,
-                    "transportType": "walk",
-                    "lineId": 10,
-                    "lineColor": "FF0000",
-                    "isDefault": True,
-                }
-            ],
-        },
-        "newbuilding": {"id": 0, "name": ""},
-        "phones": [{"number": "9012345678", "countryCode": "7"}],
-        "userId": 13332612,
-        "publishedUserId": 13332612,
-        "isByHomeowner": True,
-        "isPro": False,
-        "isRosreestrChecked": True,
-        "description": "Продаётся отличная квартира с евроремонтом",
-        "descriptionMinhash": "abc123def456abc1",
-        "addedTimestamp": 1716451200,
-        "photos": [{"fullUrl": "https://images.cdn-cian.ru/images/328739308-1.jpg"}],
-    }
-
-
-def _newbuilding_offer() -> dict[str, Any]:
-    """Новостройка без части house в geo.address[] — жмёт _recover_newbuilding_house."""
-    return {
-        "cianId": 411112222,
-        "id": 411112222,
-        "fullUrl": "https://ekb.cian.ru/sale/flat/411112222/",
-        "offerType": "flat",
-        "flatType": "rooms",
-        "isApartments": False,
-        "roomsCount": 1,
-        "totalArea": "38.0",
-        "floorNumber": 12,
-        "building": {
-            "floorsCount": 25,
-            "materialType": "monolith",
-            "buildYear": None,
-            "deadline": {"year": 2026},
-        },
-        "buildingCadastralNumber": "66:41:0204016:11396",
-        "bargainTerms": {"priceRur": 5200000, "price": 5200000, "mortgageAllowed": True},
-        "geo": {
-            "coordinates": {"lat": 56.8000, "lng": 60.6000},
-            "address": [
-                {"type": "location", "fullName": "Свердловская область"},
-                {"type": "street", "fullName": "улица Новая"},
-            ],
-        },
-        "newbuilding": {"id": 48853, "name": "ЖК Парковый квартал 29А"},
-        "isFromDeveloper": True,
-        "photos": [],
-    }
-
-
-def _studio_offer() -> dict[str, Any]:
-    return {
-        "cianId": 500011111,
-        "id": 500011111,
-        "offerType": "flat",
-        "flatType": "studio",
-        "roomsCount": None,
-        "totalArea": "24.5",
-        "floorNumber": 3,
-        "building": {"floorsCount": 9, "materialType": "panel", "buildYear": 1980},
-        "bargainTerms": {"priceRur": 3100000, "price": 3100000},
-        "geo": {
-            "coordinates": {"lat": 56.81, "lng": 60.61},
-            "address": [
-                {"type": "street", "fullName": "улица Ленина"},
-                {"type": "house", "fullName": "5"},
-            ],
-        },
-        "newbuilding": {"id": 0},
-        "photos": [],
-    }
-
-
-_OFFERS = [_vtorichka_offer(), _newbuilding_offer(), _studio_offer()]
-
-
-def _make_old() -> OldCianScraper:
-    with patch("app.services.scrapers.cian.get_scraper_delay", return_value=5.0):
-        return OldCianScraper()
-
-
-def _make_new() -> NewCianScraper:
-    return NewCianScraper(config=_CONFIG)
-
-
-def _dump(lots: list[Any]) -> list[dict[str, Any]]:
-    return [lot.model_dump() for lot in lots]
-
-
-# ── SERP _parse_serp_html parity ─────────────────────────────────────────────
-
-
-def test_serp_parse_serp_html_parity() -> None:
-    """Redux offers → list[ScrapedLot] идентичны (вторичка + новостройка + студия)."""
-    html = _serp_html(_OFFERS)
-    old_lots = _make_old()._parse_serp_html(html)
-    new_lots = _make_new()._parse_serp_html(html)
-
-    assert len(old_lots) == len(new_lots) == 3
-    assert _dump(old_lots) == _dump(new_lots)
-
-
-def test_serp_offer_to_lot_parity_per_offer() -> None:
-    """Пооферный _offer_to_lot: model_dump идентичен для каждого offer."""
-    old, new = _make_old(), _make_new()
-    for offer in _OFFERS:
-        old_lot = old._offer_to_lot(offer)
-        new_lot = new._offer_to_lot(offer)
-        assert (old_lot is None) == (new_lot is None)
-        if old_lot is not None and new_lot is not None:
-            assert old_lot.model_dump() == new_lot.model_dump()
-
-
-def test_serp_extract_total_offers_parity() -> None:
-    html = _serp_html(_OFFERS)
-    assert _make_old()._extract_total_offers(html) == _make_new()._extract_total_offers(html)
-
-
-def test_format_address_parity() -> None:
-    """_format_address (pure) — вкл. восстановление дома для ЖК без части house."""
-    building = {"houseNumber": None}
-    nb = {"name": "ЖК Парковый квартал 29А к1"}
-    cases = [
-        # (address_parts, is_newbuilding kwargs)
-        ([], {}),
-        (
-            [
-                {"type": "location", "fullName": "Свердловская область"},
-                {"type": "street", "fullName": "улица Мира"},
-                {"type": "house", "fullName": "12"},
-            ],
-            {},
-        ),
-        (
-            [
-                {"type": "location", "fullName": "Свердловская область"},
-                {"type": "street", "fullName": "улица Новая"},
-            ],
-            {
-                "building": building,
-                "building_cadastral_number": "66:41:0204016:11396",
-                "newbuilding": nb,
-                "is_newbuilding": True,
-            },
-        ),
-    ]
-    for parts, kwargs in cases:
-        assert old_format_address(parts, **kwargs) == new_format_address(parts, **kwargs)
-
-
 def test_extract_state_parity() -> None:
     """app.extract_state ≡ scraper_kit.extract_state на SERP-фикстуре."""
-    html = _serp_html(_OFFERS)
+    html = _serp_html_for_state({"results": {"totalOffers": 1, "offers": []}})
     old_state = old_extract_state(html, mfe="frontend-serp", key="initialState")
     new_state = new_extract_state(html, mfe="frontend-serp", key="initialState")
     assert old_state == new_state
     assert new_state is not None
 
 
-# ── detail fetch_detail parity ───────────────────────────────────────────────
+# ── valuation _parse_valuation_state parity (живой cian_valuation.py) ────────
-
-
-def _detail_html() -> str:
-    """defaultState + bti sister → _cianConfig['frontend-offer-card'] push HTML."""
-    default_state = {
-        "offerData": {
-            "offer": {
-                "cianId": 327830237,
-                "windowsViewType": "yardAndStreet",
-                "separateWcsCount": 1,
-                "combinedWcsCount": 0,
-                "building": {"ceilingHeight": 2.65},
-                "repairType": "cosmetic",
-                "kitchenArea": "11.5",
-                "description": "Отличная квартира",
-                "priceChanges": [
-                    {"changeTime": "2026-04-01T00:00:00Z", "price": 5500000, "diffPercent": -1.8}
-                ],
-                "agent": {
-                    "id": 7654321,
-                    "companyName": "Первичка Плюс",
-                    "isPro": True,
-                    "skills": [],
-                    "accountType": "specialist",
-                },
-                "newbuilding": {"id": 0},
-            },
-            "stats": {
-                "totalViewsFormattedString": "1 042",
-                "todayViewsFormattedString": "7",
-            },
-        }
-    }
-    bti_state = {"houseData": {"series": "1-464", "wallMaterial": "brick"}}
-    ds_json = json.dumps(default_state, ensure_ascii=False)
-    bti_json = json.dumps(bti_state, ensure_ascii=False)
-    return (
-        "<html><body><script>\n"
-        "window._cianConfig['frontend-offer-card'] = "
-        "window._cianConfig['frontend-offer-card'] || [];\n"
-        "window._cianConfig['frontend-offer-card'].push({"
-        f"key: 'defaultState', value: {ds_json}, priority: 10, filter: null"
-        "});\n"
-        "window._cianConfig['frontend-offer-card'].push({"
-        f"key: 'bti', value: {bti_json}, priority: 5, filter: null"
-        "});\n</script></body></html>"
-    )
-
-
-def _mock_session(html: str) -> MagicMock:
-    response = MagicMock()
-    response.status_code = 200
-    response.text = html
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-    return session
-
-
-@pytest.mark.asyncio
-async def test_detail_fetch_parity() -> None:
-    """fetch_detail (defaultState + bti) → DetailEnrichment идентичен (все поля)."""
-    html = _detail_html()
-    url = "https://ekb.cian.ru/sale/flat/327830237/"
-
-    old_res = await old_fetch_detail(url, session=_mock_session(html))
-    new_res = await new_fetch_detail(url, session=_mock_session(html))
-
-    assert old_res is not None
-    assert new_res is not None
-    assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)
-
-
-# ── valuation _parse_valuation_state parity ──────────────────────────────────
 
 
 def _valuation_state() -> dict[str, Any]:
@@ -595,7 +312,7 @@ def test_kit_cian_session_config_requirement_asymmetry() -> None:
     db2.execute.assert_called_once()
 
 
-# ── newbuilding pure-экстракторы parity ──────────────────────────────────────
+# ── newbuilding pure-экстракторы parity (живой cian_newbuilding.py) ──────────
 
 
 def test_newbuilding_extractors_parity() -> None:

tradein-mvp/backend/tests/test_scraper_kit_domclick_golden_parity.py

@@ -1,183 +0,0 @@
-"""Golden-parity: `scraper_kit.providers.domclick.serp` ≡ `app.services.scrapers.domclick`.
-
-Strangler-инвариант (#2133): новая scraper_kit-копия domclick-провайдера должна
-давать БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
-Развязка (get_scraper_delay→delay_provider, base/exceptions/repair-normalizer→
-scraper_kit.*) НЕ меняет распарсенные данные.
-
-Гоняем ОБА модуля на одной BFF-JSON-фикстуре и сравниваем:
-  - `_extract_json` (JSON из HTML-обёртки + QRATOR-block detection);
-  - `_map_item` (offer → ScrapedLot): студия rooms=0, sentinel 0→None, agency;
-  - `_is_geo_ok` (bbox + offerRegionName guard);
-  - url-билдеры и чистые хелперы (_build_offers_url/_build_count_url/
-    _parse_publish_date/_extract_agency_name).
-
-Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
-"""
-
-from __future__ import annotations
-
-import json
-import os
-from pathlib import Path
-from types import SimpleNamespace
-from typing import Any
-from unittest.mock import patch
-
-# Старый app.services.scrapers.domclick тянет app.core.config.settings=Settings(),
-# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-# НОВЫЙ (scraper_kit) — ruff сортирует scraper_kit (first-party) выше app-импортов.
-from scraper_kit.providers.domclick.serp import DomClickScraper as NewDomClickScraper
-from scraper_kit.providers.domclick.serp import _build_count_url as new_build_count_url
-from scraper_kit.providers.domclick.serp import _build_offers_url as new_build_offers_url
-from scraper_kit.providers.domclick.serp import _extract_agency_name as new_extract_agency
-from scraper_kit.providers.domclick.serp import _extract_json as new_extract_json
-from scraper_kit.providers.domclick.serp import _parse_publish_date as new_parse_publish_date
-
-from app.services.scrapers.domclick import DomClickScraper as OldDomClickScraper
-from app.services.scrapers.domclick import _build_count_url as old_build_count_url
-from app.services.scrapers.domclick import _build_offers_url as old_build_offers_url
-from app.services.scrapers.domclick import _extract_agency_name as old_extract_agency
-from app.services.scrapers.domclick import _extract_json as old_extract_json
-from app.services.scrapers.domclick import _parse_publish_date as old_parse_publish_date
-
-_FIXTURES = Path(__file__).parent / "fixtures"
-_BFF_JSON = (_FIXTURES / "domclick_bff_offers_sample.json").read_text(encoding="utf-8")
-_BFF_ITEMS: list[dict[str, Any]] = json.loads(_BFF_JSON)["result"]["items"]
-
-
-def _make_old() -> OldDomClickScraper:
-    """Старый DomClickScraper без DB: patch get_scraper_delay."""
-    with patch("app.services.scrapers.domclick.get_scraper_delay", return_value=8.0):
-        return OldDomClickScraper()
-
-
-def _make_new() -> NewDomClickScraper:
-    """Новый DomClickScraper с инжектируемым ScraperConfig + delay_provider.
-
-    Парсинг-тесты не дёргают fetch_city → browser_http_endpoint не используется,
-    но конструктор требует config (duck-typed namespace достаточен).
-    """
-    config = SimpleNamespace(browser_http_endpoint="http://tradein-browser:9000")
-    return NewDomClickScraper(config, delay_provider=lambda _name: 8.0)
-
-
-# ── _extract_json parity ──────────────────────────────────────────────────────
-
-
-def test_extract_json_parity_bare() -> None:
-    """_extract_json на bare JSON-теле идентичен."""
-    assert old_extract_json(_BFF_JSON) == new_extract_json(_BFF_JSON)
-
-
-def test_extract_json_parity_pre_wrapped() -> None:
-    """_extract_json на <pre>-обёрнутом JSON идентичен (реальная форма ответа)."""
-    wrapped = f"<html><body><pre>{_BFF_JSON}</pre></body></html>"
-    assert old_extract_json(wrapped) == new_extract_json(wrapped)
-
-
-def test_extract_json_parity_qrator_block() -> None:
-    """QRATOR block-страница → DomClickBlockedError в обоих (идентичное поведение)."""
-    from scraper_kit.domclick_exceptions import DomClickBlockedError as NewBlocked
-
-    from app.services.scrapers.domclick_exceptions import DomClickBlockedError as OldBlocked
-
-    block_html = "<html><body>Система защиты QRATOR: captcha</body></html>"
-    old_raised = new_raised = False
-    try:
-        old_extract_json(block_html)
-    except OldBlocked:
-        old_raised = True
-    try:
-        new_extract_json(block_html)
-    except NewBlocked:
-        new_raised = True
-    assert old_raised and new_raised
-
-
-# ── _map_item parity (offer → ScrapedLot) ─────────────────────────────────────
-
-
-def test_map_item_parity_all_items() -> None:
-    """Каждый offer-item → ScrapedLot идентичен (студия rooms=0, sentinel 0→None)."""
-    old = _make_old()
-    new = _make_new()
-    for item in _BFF_ITEMS:
-        # force_rooms=0 для студий (st-бакет) — проверяем обе ветки маппинга.
-        for force_rooms in (None, 0):
-            old_lot = old._map_item(item, force_rooms=force_rooms)
-            new_lot = new._map_item(item, force_rooms=force_rooms)
-            if old_lot is None or new_lot is None:
-                assert old_lot is None and new_lot is None
-                continue
-            assert old_lot.model_dump() == new_lot.model_dump()
-    # Счётчик parse-ошибок эволюционирует одинаково.
-    assert old.parse_failures == new.parse_failures
-
-
-# ── _is_geo_ok parity (bbox + region guard) ───────────────────────────────────
-
-
-def test_is_geo_ok_parity_all_items() -> None:
-    """Гео-гард (offerRegionName + bbox) идентичен на всех items."""
-    old = _make_old()
-    new = _make_new()
-    for item in _BFF_ITEMS:
-        assert old._is_geo_ok(item) == new._is_geo_ok(item)
-
-
-# ── url-билдеры + чистые хелперы ──────────────────────────────────────────────
-
-
-def test_url_builders_parity() -> None:
-    """_build_offers_url / _build_count_url идентичны (вкл. '5+' → '5%2B')."""
-    cases: list[tuple[str, int | None, int | None]] = [
-        ("st", None, None),
-        ("2", 3_000_000, 8_000_000),
-        ("5+", 15_000_000, None),
-        ("1", None, 5_000_000),
-    ]
-    for rooms, gte, lte in cases:
-        assert old_build_count_url(rooms, gte, lte) == new_build_count_url(rooms, gte, lte)
-        for offset in (0, 20, 1980):
-            assert old_build_offers_url(rooms, gte, lte, offset) == new_build_offers_url(
-                rooms, gte, lte, offset
-            )
-
-
-def test_parse_publish_date_parity() -> None:
-    """_parse_publish_date идентичен на валидных/битых ISO-строках."""
-    for iso in [None, "", "2026-06-28T09:15:00+05:00", "2026-06-27", "not-a-date", "2026-13-40"]:
-        assert old_parse_publish_date(iso) == new_parse_publish_date(iso)
-
-
-def test_extract_agency_name_parity() -> None:
-    """_extract_agency_name идентичен на строковых/dict/пустых seller-блоках."""
-    sellers: list[dict[str, Any]] = [
-        {},
-        {"company": "АН Этажи", "agent": "Иванов"},
-        {"company": "   ", "agent": {"name": "Резерв"}},
-        {"agent": {"fullName": "Петров П."}},
-        {"company": {"title": "СберРиелт"}},
-        {"agent": 12345},
-    ]
-    for seller in sellers:
-        assert old_extract_agency(seller) == new_extract_agency(seller)
-
-
-# ── strangler-guard: kit-провайдер не импортирует app.* ───────────────────────
-
-
-def test_kit_domclick_has_no_app_imports() -> None:
-    """Ни один модуль scraper_kit.providers.domclick.* не импортирует app.* (развязка)."""
-    import inspect
-
-    from scraper_kit.providers.domclick import serp
-
-    source = inspect.getsource(serp)
-    for line in source.splitlines():
-        stripped = line.strip()
-        assert not stripped.startswith("from app"), f"serp: {line!r}"
-        assert not stripped.startswith("import app"), f"serp: {line!r}"

tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py

@@ -32,16 +32,21 @@ Migrates legacy scraper imports of these files to their `scraper_kit` equivalent
 
 This file proves the DELTA introduced by the #2310 migration specifically —
 general avito/cian-provider parsing golden-parity is already covered by
-`tests/test_scraper_kit_avito_golden_parity.py` and
+`tests/test_scraper_kit_avito_golden_parity.py` and (while it existed)
 `tests/test_scraper_kit_cian_golden_parity.py`. Uses `tests/support/parity.assert_parity`
 per the epic's parity-gate convention (issue #2304).
+
+`test_cian_fetch_detail_via_browser_fetcher_parity` (legacy `app.services.scrapers.
+cian_detail.fetch_detail` vs kit, browser_fetcher call shape) removed (#2397 Part E2)
+— `app.services.scrapers.cian_detail` deleted, 0 runtime importers (cian_history_backfill.py
+already calls the kit `fetch_detail` exclusively, proven by
+`test_cian_history_backfill_browser_fetcher_uses_settings_endpoint` below).
 """
 
 from __future__ import annotations
 
 import asyncio
 import dataclasses
-import json
 import os
 from typing import Any
 from unittest.mock import AsyncMock, MagicMock, patch
@@ -53,12 +58,10 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/
 from scraper_kit.avito_exceptions import AvitoBlockedError as KitAvitoBlockedError
 from scraper_kit.providers.avito.detail import fetch_detail as kit_avito_fetch_detail
 from scraper_kit.providers.avito.serp import AvitoScraper as KitAvitoScraper
-from scraper_kit.providers.cian.detail import fetch_detail as kit_cian_fetch_detail
 
 from app.core.config import settings
 from app.services.scraper_adapters import RealScraperConfig
 from app.services.scrapers.avito_detail import fetch_detail as legacy_avito_fetch_detail
-from app.services.scrapers.cian_detail import fetch_detail as legacy_cian_fetch_detail
 from tests.support.parity import assert_parity
 
 # ── avito fetch_detail: backconnect config-gate (avito_detail_backfill.py delta) ──
@@ -144,30 +147,7 @@ def test_avito_scraper_rotate_ip_reads_injected_config() -> None:
     assert scraper._config.avito_proxy_rotate_url == settings.avito_proxy_rotate_url
 
 
-# ── cian fetch_detail via browser_fetcher (cian_history_backfill.py call shape) ──
+# ── avito fetch_detail via browser_fetcher (avito_detail_backfill.py browser_mode) ──
-
-
-def _cian_detail_html() -> str:
-    default_state = {
-        "offerData": {
-            "offer": {
-                "cianId": 777,
-                "priceChanges": [
-                    {"changeTime": "2026-06-01T00:00:00Z", "price": 5900000, "diffPercent": -0.5}
-                ],
-            },
-            "stats": {},
-        }
-    }
-    ds_json = json.dumps(default_state, ensure_ascii=False)
-    return (
-        "<html><body><script>\n"
-        "window._cianConfig['frontend-offer-card'] = "
-        "window._cianConfig['frontend-offer-card'] || [];\n"
-        "window._cianConfig['frontend-offer-card'].push({"
-        f"key: 'defaultState', value: {ds_json}, priority: 10, filter: null"
-        "});\n</script></body></html>"
-    )
 
 
 def _fake_browser_fetcher(html: str) -> Any:
@@ -176,30 +156,6 @@ def _fake_browser_fetcher(html: str) -> Any:
     return bf
 
 
-def test_cian_fetch_detail_via_browser_fetcher_parity() -> None:
-    """cian_history_backfill.py's listings block ALWAYS calls fetch_detail(url,
-    browser_fetcher=bf) — never the curl_cffi/config own-session path exercised by
-    tests/test_scraper_kit_pricehistory_session_parity.py (cian_price_history.py's
-    call shape). Proves parity holds for THIS call shape too."""
-    html = _cian_detail_html()
-    url = "https://ekb.cian.ru/sale/flat/777/"
-
-    def _run_legacy() -> Any:
-        return dataclasses.asdict(
-            asyncio.run(legacy_cian_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
-        )
-
-    def _run_kit() -> Any:
-        return dataclasses.asdict(
-            asyncio.run(kit_cian_fetch_detail(url, browser_fetcher=_fake_browser_fetcher(html)))
-        )
-
-    assert_parity(legacy_fn=_run_legacy, kit_fn=_run_kit, fixtures=[()])
-
-
-# ── avito fetch_detail via browser_fetcher (avito_detail_backfill.py browser_mode) ──
-
-
 def test_avito_fetch_detail_via_browser_fetcher_parity() -> None:
     """browser_mode branch of avito_detail_backfill.py calls fetch_detail(url,
     browser_fetcher=browser_fetcher) — config is never read on this path (early

tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py

@@ -17,32 +17,31 @@ cian-provider / ScrapedLot golden-parity is already covered by
 `tests/test_scraper_kit_cian_golden_parity.py` and
 `tests/test_scraper_kit_base_golden_parity.py`. Uses `tests/support/parity.assert_parity`
 per the epic's parity-gate convention (issue #2304).
+
+`test_fetch_detail_with_config_injection_parity` / `test_fetch_detail_own_session_proxy_
+wiring_parity` (legacy `app.services.scrapers.cian_detail.fetch_detail` vs kit) removed
+(#2397 Part E2) — `cian_detail` deleted, 0 runtime importers. `extract_state`
+(`cian_state_parser`, still legacy-live) and `ScrapedLot` (`base`, still legacy-live)
+parity below is unaffected.
 """
 
 from __future__ import annotations
 
-import asyncio
-import dataclasses
 import json
 import os
 from typing import Any
-from unittest.mock import AsyncMock, MagicMock, patch
 
-# Старый app.services.scrapers.cian* / app.core.config.settings требует DATABASE_URL
+# Старый app.services.scrapers.cian_state_parser/base / app.core.config.settings
-# при импорте (тот же паттерн, что в test_scraper_kit_cian_golden_parity.py).
+# требует DATABASE_URL при импорте (тот же паттерн, что в
+# test_scraper_kit_cian_golden_parity.py).
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
 
 # Kit — то, на что теперь ссылаются мигрированные файлы (#2306).
 from scraper_kit.base import ScrapedLot as KitScrapedLot
 from scraper_kit.cian_state_parser import extract_state as kit_extract_state
-from scraper_kit.providers.cian.detail import fetch_detail as kit_fetch_detail
 
-from app.core.config import settings
+# Legacy — эталон сравнения (эти модули НЕ меняются в #2306, остаются live).
-from app.services.scraper_adapters import RealScraperConfig
-
-# Legacy — эталон сравнения (эти модули НЕ меняются в #2306).
 from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot
-from app.services.scrapers.cian_detail import fetch_detail as legacy_fetch_detail
 from app.services.scrapers.cian_state_parser import extract_state as legacy_extract_state
 from tests.support.parity import assert_parity
 
@@ -108,113 +107,3 @@ def test_scraped_lot_model_dump_parity() -> None:
         kit_fn=lambda kw: KitScrapedLot(**kw).model_dump(),
         fixtures=[kwargs],
     )
-
-
-# ── fetch_detail / DetailEnrichment parity (cian_price_history.py dependency) ──
-
-
-def _detail_html() -> str:
-    default_state = {
-        "offerData": {
-            "offer": {
-                "cianId": 555111,
-                "priceChanges": [
-                    {"changeTime": "2026-05-01T00:00:00Z", "price": 6100000, "diffPercent": -1.2}
-                ],
-            },
-            "stats": {},
-        }
-    }
-    ds_json = json.dumps(default_state, ensure_ascii=False)
-    return (
-        "<html><body><script>\n"
-        "window._cianConfig['frontend-offer-card'] = "
-        "window._cianConfig['frontend-offer-card'] || [];\n"
-        "window._cianConfig['frontend-offer-card'].push({"
-        f"key: 'defaultState', value: {ds_json}, priority: 10, filter: null"
-        "});\n</script></body></html>"
-    )
-
-
-def _mock_session(html: str) -> MagicMock:
-    response = MagicMock()
-    response.status_code = 200
-    response.text = html
-    session = MagicMock()
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-    return session
-
-
-def _run_legacy_fetch(url: str, html: str) -> Any:
-    return dataclasses.asdict(asyncio.run(legacy_fetch_detail(url, session=_mock_session(html))))
-
-
-def _run_kit_fetch(url: str, html: str) -> Any:
-    # Точно тот вызов, что теперь делает cian_price_history.backfill_cian_price_history:
-    # config=RealScraperConfig() добавлен по миграции (#2306); session= общий mock здесь
-    # только чтобы избежать сетевого I/O — сам config не участвует в session-пути, так
-    # что этот тест доказывает: инжекция config= не меняет результат парсинга.
-    result = asyncio.run(
-        kit_fetch_detail(url, config=RealScraperConfig(), session=_mock_session(html))
-    )
-    return dataclasses.asdict(result)
-
-
-def test_fetch_detail_with_config_injection_parity() -> None:
-    html = _detail_html()
-    url = "https://ekb.cian.ru/sale/flat/555111/"
-    assert_parity(
-        legacy_fn=_run_legacy_fetch,
-        kit_fn=_run_kit_fetch,
-        fixtures=[(url, html)],
-    )
-
-
-def test_fetch_detail_own_session_proxy_wiring_parity(monkeypatch: Any) -> None:
-    """cian_price_history calls fetch_detail WITHOUT session= (own-session curl_cffi path).
-
-    Kit fetch_detail without config= silently drops the proxy (direct connection) — a
-    real behaviour regression risk (#806: datacenter-IP gets banned by Cian without a
-    proxy). This proves config=RealScraperConfig() (our actual call site) constructs
-    AsyncSession(proxies=...) identically to the legacy direct-settings path.
-
-    #2362 (F4b): kit's own-session branch now builds the session via
-    `scraper_kit.providers._base.build_curl_cffi_session()` instead of constructing
-    `AsyncSession` directly in `detail.py` — so the spy patches `AsyncSession` where
-    the constructor call now actually happens (`_base`), not in `detail` anymore.
-    The asserted kwargs (proxies=...) are unchanged.
-    """
-    monkeypatch.setattr(settings, "cian_proxy_url_env", "http://test-proxy.local:8080")
-
-    captured: dict[str, dict[str, Any]] = {}
-
-    def _make_fake_session(name: str) -> Any:
-        class _FakeSession:
-            def __init__(self, **kwargs: Any) -> None:
-                captured[name] = kwargs
-
-            async def get(self, *_args: Any, **_kwargs: Any) -> Any:
-                response = MagicMock()
-                response.status_code = 200
-                response.text = _detail_html()
-                return response
-
-            async def close(self) -> None:
-                pass
-
-        return _FakeSession
-
-    url = "https://ekb.cian.ru/sale/flat/1/"
-
-    with patch("app.services.scrapers.cian_detail.AsyncSession", _make_fake_session("legacy")):
-        asyncio.run(legacy_fetch_detail(url))
-
-    with patch("scraper_kit.providers._base.AsyncSession", _make_fake_session("kit")):
-        asyncio.run(kit_fetch_detail(url, config=RealScraperConfig()))
-
-    assert captured["legacy"].get("proxies") == captured["kit"].get("proxies")
-    assert captured["kit"]["proxies"] == {
-        "http": "http://test-proxy.local:8080",
-        "https": "http://test-proxy.local:8080",
-    }

tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py

@@ -1,4 +1,5 @@
-"""Golden-parity: `scraper_kit.providers.yandex.*` парсинг ≡ `app.services.scrapers.yandex_*`.
+"""Golden-parity: `scraper_kit.providers.yandex.*` парсинг ≡ ещё живым
+`app.services.scrapers.yandex_*`.
 
 Strangler-инвариант (#2133): новая scraper_kit-копия yandex-провайдера должна давать
 БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
@@ -6,34 +7,34 @@ Strangler-инвариант (#2133): новая scraper_kit-копия yandex-
 house_type_normalizer→providers.yandex.shared, base/browser_fetcher/helpers→scraper_kit.*)
 НЕ меняет распарсенные данные.
 
-Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
+`app.services.scrapers.yandex_realty` удалён (#2397 Part E2, 0 runtime-импортёров).
-  - SERP `_entity_to_lot` (gate-API entity → ScrapedLot) на репрезентативных entity;
+SERP `_entity_to_lot`/`_parse_gate_json`/`_build_url` parity-тесты (сравнение с
-  - SERP `_parse_gate_json` (payload → list[ScrapedLot]) с новостройкой/вторичкой;
+`YandexRealtyScraper`) удалены вместе с ним (легаси-эталон исчез).
-  - `normalize_house_type` (SCREAMING/camelCase → канон);
-  - newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) на синтетике;
-  - `_build_url` (gate-API URL builder) — детерминированность.
 
-Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
+Оставшееся сравнивает kit ≡ ЕЩЁ ЖИВЫЕ легаси-модули:
+  - `normalize_house_type` — `app.services.scrapers.house_type_normalizer` (живой);
+  - newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) —
+    `app.services.scrapers.yandex_newbuilding` (живой);
+  - strangler-guard: kit-провайдер не импортирует `app.*`.
 
 Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
 финальный шаг scraper_kit-миграции, 0 runtime importers) — detail/valuation
-parity-проверки убраны отсюда вместе с ними. Kit-side coverage для detail/valuation
+parity-проверки убраны отсюда ранее. Kit-side coverage для detail/valuation
 `.parse()` сохранена без изменений в `tests/test_yandex_detail.py` /
 `tests/test_yandex_detail_structural.py` / `tests/test_yandex_valuation.py` /
 `tests/test_yandex_valuation_save.py` (переведены на kit-эквиваленты тем же PR).
-`yandex_realty` / `yandex_newbuilding` / `house_type_normalizer` остаются легаси
+
-(не мигрированы) — их parity-проверки не тронуты.
+Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
 """
 
 from __future__ import annotations
 
 import inspect
 import os
-from types import SimpleNamespace
-from typing import Any
 
-# Старый app.services.scrapers.yandex_* импортирует app.core.config.settings=Settings(),
+# Старый app.services.scrapers.house_type_normalizer/yandex_newbuilding импортирует
-# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
+# app.core.config.settings=Settings(), которому нужен DATABASE_URL. Офлайн-парсинг
+# БД не трогает — фиктивный DSN достаточен.
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
 
 # НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) yandex-парсеры — оба в одном
@@ -41,9 +42,6 @@ os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/
 from scraper_kit.providers.yandex.newbuilding import (
     YandexNewbuildingScraper as NewNewbuildingScraper,
 )
-from scraper_kit.providers.yandex.serp import YandexRealtyScraper as NewRealtyScraper
-from scraper_kit.providers.yandex.serp import _entity_to_lot as new_entity_to_lot
-from scraper_kit.providers.yandex.serp import _parse_gate_json as new_parse_gate_json
 from scraper_kit.providers.yandex.shared import normalize_house_type as new_normalize_house_type
 
 from app.services.scrapers.house_type_normalizer import (
@@ -52,192 +50,6 @@ from app.services.scrapers.house_type_normalizer import (
 from app.services.scrapers.yandex_newbuilding import (
     YandexNewbuildingScraper as OldNewbuildingScraper,
 )
-from app.services.scrapers.yandex_realty import YandexRealtyScraper as OldRealtyScraper
-from app.services.scrapers.yandex_realty import _entity_to_lot as old_entity_to_lot
-from app.services.scrapers.yandex_realty import _parse_gate_json as old_parse_gate_json
-
-# Инжектируемый конфиг для kit-скрапперов (duck-typed namespace, structurally
-# удовлетворяет scraper_kit.contracts.ScraperConfig в объёме читаемых полей).
-_KIT_CONFIG = SimpleNamespace(
-    scraper_proxy_url=None,
-    yandex_proxy_rotate_url=None,
-    avito_proxy_rotate_url=None,
-)
-
-
-# ── Fixtures: gate-API entities (mirror test_yandex_realty_serp) ─────────────
-
-_ENTITY_FULL: dict = {
-    "offerId": 4740475460451078271,
-    "url": "//realty.yandex.ru/offer/4740475460451078271",
-    "price": {"currency": "RUR", "value": 4500000, "valuePerPart": 119048},
-    "area": {"value": 37.8, "unit": "SQUARE_METER"},
-    "livingSpace": {"value": 20, "unit": "SQUARE_METER"},
-    "kitchenSpace": {"value": 12.7, "unit": "SQUARE_METER"},
-    "roomsTotal": 1,
-    "floorsOffered": [8],
-    "floorsTotal": 25,
-    "ceilingHeight": 2.7,
-    "building": {
-        "builtYear": 2016,
-        "buildingType": "MONOLIT",
-        "siteId": 280938,
-        "siteName": "Peremen",
-    },
-    "location": {
-        "geocoderAddress": "Ekaterinburg, ulitsa Evgenia Savkova, 8",
-        "point": {"latitude": 56.79512, "longitude": 60.49186},
-    },
-    "mainImages": [
-        "//avatars.mds.yandex.net/get-realty-offers/11904018/abc/main",
-        "//avatars.mds.yandex.net/get-realty-offers/14717586/def/main",
-    ],
-}
-
-_ENTITY_STUDIO: dict = {
-    "offerId": 9999999,
-    "url": "//realty.yandex.ru/offer/9999999",
-    "price": {"value": 3200000, "valuePerPart": 114285},
-    "area": {"value": 28.0},
-    "livingSpace": None,
-    "kitchenSpace": None,
-    "roomsTotal": None,  # studio -> rooms=0
-    "floorsOffered": [2],
-    "floorsTotal": 9,
-    "ceilingHeight": None,
-    "building": {"builtYear": 2010, "buildingType": "PANEL", "siteId": None, "siteName": None},
-    "location": {
-        "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 5",
-        "point": {"latitude": 56.838, "longitude": 60.597},
-    },
-    "mainImages": [],
-}
-
-_ENTITY_RICH_OWNER: dict = {
-    "offerId": 5500000000000000001,
-    "url": "//realty.yandex.ru/offer/5500000000000000001",
-    "creationDate": "2026-05-24T15:10:27Z",
-    "description": "Продаётся светлая квартира с видом на парк.",
-    "author": {"category": "OWNER", "agentName": "Иван", "organization": None},
-    "price": {"value": 7200000, "valuePerPart": 150000, "trend": "DECREASED", "previous": 7500000},
-    "predictions": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}},
-    "area": {"value": 48.0},
-    "roomsTotal": 2,
-    "floorsOffered": [5],
-    "floorsTotal": 12,
-    "building": {"builtYear": 2018, "buildingType": "MONOLIT"},
-    "location": {
-        "geocoderAddress": "Ekaterinburg, ulitsa Mira, 10",
-        "point": {"latitude": 56.84, "longitude": 60.61},
-        "metroList": [
-            {"name": "Ploshchad 1905 goda", "timeToMetro": 7},
-            {"name": "Geologicheskaya", "timeToMetro": 12},
-        ],
-    },
-    "mainImages": [],
-}
-
-_ENTITY_RICH_AGENCY: dict = {
-    "offerId": 5500000000000000002,
-    "url": "//realty.yandex.ru/offer/5500000000000000002",
-    "creationDate": "2026-06-01T09:00:00Z",
-    "description": "Агентская продажа.",
-    "author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"},
-    "price": {"value": 9900000, "trend": "UNCHANGED"},
-    "area": {"value": 60.0},
-    "roomsTotal": 3,
-    "floorsOffered": [3],
-    "floorsTotal": 9,
-    "building": {"builtYear": 2012, "buildingType": "BRICK"},
-    "location": {
-        "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1",
-        "point": {"latitude": 56.83, "longitude": 60.6},
-        "metro": {"name": "Dinamo", "timeToMetro": 4},
-    },
-    "mainImages": [],
-}
-
-# #2235: novostroyka offer whose gate-API `url` is a developer/redirect landing
-# (shared by many flats). Both parsers must canonicalise it to offer_id-based url.
-_ENTITY_PARTNER_URL: dict = {
-    "offerId": 7416316697684447350,
-    "url": "http://na100.pro/go.php?link=LtWgA6kWd3xjlrUBCLl6VGjo",
-    "price": {"value": 7790000, "valuePerPart": 130000},
-    "area": {"value": 60.0},
-    "roomsTotal": 2,
-    "floorsOffered": [5],
-    "floorsTotal": 16,
-    "building": {"builtYear": 2024, "buildingType": "MONOLIT", "siteId": 999001},
-    "location": {
-        "geocoderAddress": "Ekaterinburg, ulitsa Titova, 1",
-        "point": {"latitude": 56.8, "longitude": 60.6},
-    },
-    "mainImages": [],
-}
-
-_ALL_ENTITIES = [
-    _ENTITY_FULL,
-    _ENTITY_STUDIO,
-    _ENTITY_RICH_OWNER,
-    _ENTITY_RICH_AGENCY,
-    _ENTITY_PARTNER_URL,
-]
-
-
-def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
-    if pager is None:
-        pager = {"page": 0, "pageSize": 20, "totalItems": len(entities), "totalPages": 1}
-    return {"response": {"search": {"offers": {"entities": entities, "pager": pager}}}}
-
-
-def _dump_lots(lots: list[Any]) -> list[dict[str, Any]]:
-    return [lot.model_dump() for lot in lots]
-
-
-# ── SERP _entity_to_lot parity ───────────────────────────────────────────────
-
-
-def test_entity_to_lot_parity_all_entities() -> None:
-    """gate-API entity → ScrapedLot идентичен на всех репрезентативных entity."""
-    for entity in _ALL_ENTITIES:
-        old_lot = old_entity_to_lot(entity)
-        new_lot = new_entity_to_lot(entity)
-        assert (old_lot is None) == (new_lot is None)
-        assert old_lot is not None and new_lot is not None
-        assert old_lot.model_dump() == new_lot.model_dump()
-
-
-def test_entity_to_lot_parity_new_flat_segments() -> None:
-    """new_flat=YES/NO → listing_segment идентичен (новостройка/вторичка)."""
-    for new_flat in ("NO", "YES"):
-        old_lot = old_entity_to_lot(_ENTITY_FULL, new_flat=new_flat)
-        new_lot = new_entity_to_lot(_ENTITY_FULL, new_flat=new_flat)
-        assert old_lot is not None and new_lot is not None
-        assert old_lot.model_dump() == new_lot.model_dump()
-
-
-def test_entity_to_lot_parity_ceiling_out_of_range() -> None:
-    """ceilingHeight мусор (18 / 1.6) → одинаково дропается в None (#2007)."""
-    for bad_ceiling in (18, 1.6):
-        entity = {**_ENTITY_FULL, "ceilingHeight": bad_ceiling}
-        old_lot = old_entity_to_lot(entity)
-        new_lot = new_entity_to_lot(entity)
-        assert old_lot is not None and new_lot is not None
-        assert old_lot.model_dump() == new_lot.model_dump()
-
-
-# ── SERP _parse_gate_json parity ─────────────────────────────────────────────
-
-
-def test_parse_gate_json_parity() -> None:
-    """payload → list[ScrapedLot] идентичен (вторичка и новостройка)."""
-    payload = _make_gate_payload(_ALL_ENTITIES)
-    for new_flat in ("NO", "YES"):
-        old_lots = old_parse_gate_json(payload, page_param=2, new_flat=new_flat)
-        new_lots = new_parse_gate_json(payload, page_param=2, new_flat=new_flat)
-        assert len(old_lots) == len(new_lots) > 0
-        assert _dump_lots(old_lots) == _dump_lots(new_lots)
-
 
 # ── normalize_house_type parity ──────────────────────────────────────────────
 
@@ -296,23 +108,6 @@ def test_newbuilding_parse_parity() -> None:
     assert old_res.model_dump() == new_res.model_dump()
 
 
-# ── _build_url parity ────────────────────────────────────────────────────────
-
-
-def test_build_url_parity() -> None:
-    """gate-API URL builder идентичен на репрезентативных комбо."""
-    old_scraper = OldRealtyScraper()
-    new_scraper = NewRealtyScraper(config=_KIT_CONFIG)
-    combos = [
-        dict(page=1),
-        dict(page=3, rooms="2", price_min=5_000_000, price_max=7_000_000),
-        dict(page=1, rooms="studio", new_flat="YES"),
-        dict(page=2, rooms="4+", price_min=25_000_000),
-    ]
-    for combo in combos:
-        assert old_scraper._build_url(**combo) == new_scraper._build_url(**combo)
-
-
 # ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────
 
 

tradein-mvp/backend/tests/test_scraper_proxy.py

@@ -177,30 +177,13 @@ _EXPECTED_PROXIES = {"http": _PROXY_URL, "https": _PROXY_URL}
 
 
 # ── YandexRealtyScraper ──────────────────────────────────────────────────────
-
+#
-
+# `test_yandex_realty_opens_browser_fetcher` (legacy `app.services.scrapers.
-@pytest.mark.asyncio
+# yandex_realty.YandexRealtyScraper.__aenter__` opens ONE BrowserFetcher session)
-async def test_yandex_realty_opens_browser_fetcher(monkeypatch):
+# removed (#2397 Part E2) — `yandex_realty` deleted, 0 runtime importers. Kit
-    """YandexRealtyScraper.__aenter__ opens ONE BrowserFetcher (camoufox) session.
+# equivalent already covered (opens exactly one BrowserFetcher session via
-
+# `build_browser_fetcher`) by `test_kit_serp_proxy_pool.py::
-    Transport moved off the proxied curl_cffi session to BrowserFetcher (camoufox);
+# test_yandex_serp_no_pool_by_default` (and the pool-wiring variants alongside it).
-    the mobile proxy is applied server-side by the tradein-browser service, not via a
-    proxies= kwarg here.
-    """
-    from app.services.scrapers.yandex_realty import YandexRealtyScraper
-
-    mock_browser = MagicMock()
-    mock_browser.__aenter__ = AsyncMock(return_value=mock_browser)
-    mock_browser.__aexit__ = AsyncMock(return_value=None)
-
-    monkeypatch.setattr(
-        "app.services.scrapers.yandex_realty.BrowserFetcher", lambda *a, **k: mock_browser
-    )
-    scraper = YandexRealtyScraper()
-    await scraper.__aenter__()
-
-    mock_browser.__aenter__.assert_awaited_once()
-    assert scraper._browser is mock_browser
 
 
 # ── YandexValuationScraper ───────────────────────────────────────────────────

tradein-mvp/backend/tests/test_snapshot_writer.py

@@ -9,14 +9,23 @@
 5. Ошибка snapshot не прерывает основной INSERT в save_listings.
 
 Используем unittest.mock — без реальной БД.
+
+`app.services.scrapers.cian_detail` удалён (#2397 Part E2, 0 runtime-импортёров) —
+`DetailEnrichment`/`save_detail_enrichment` ретаргетированы на
+`scraper_kit.providers.cian.detail` (byte-identical strangler-копия; `snapshot_writer`
+тестируется по-прежнему через живой `app.services.scrapers.snapshot_writer` —
+kit и app-версии этого модуля идентичны, kit's `save_detail_enrichment` импортирует
+`upsert_listing_snapshot` из `scraper_kit.snapshot_writer`).
 """
+
 from __future__ import annotations
 
 from contextlib import contextmanager
 from datetime import date
 from unittest.mock import MagicMock, patch
 
-from app.services.scrapers.cian_detail import DetailEnrichment, save_detail_enrichment
+from scraper_kit.providers.cian.detail import DetailEnrichment, save_detail_enrichment
+
 from app.services.scrapers.snapshot_writer import upsert_listing_snapshot
 
 # ── Helpers ───────────────────────────────────────────────────────────────────
@@ -220,7 +229,7 @@ def test_save_detail_enrichment_snapshot_failure_does_not_abort():
     db = _mock_db_detail(price_rub=4_000_000)
 
     with patch(
-        "app.services.scrapers.cian_detail.upsert_listing_snapshot",
+        "scraper_kit.providers.cian.detail.upsert_listing_snapshot",
         side_effect=RuntimeError("db snapshot blip"),
     ):
         enrichment = DetailEnrichment(ceiling_height=2.5)
@@ -261,9 +270,9 @@ def test_save_detail_enrichment_oph_on_conflict_constraint():
     sqls = _get_all_sqls(db)
     oph_sqls = [s for s in sqls if "offer_price_history" in s]
     assert oph_sqls, "INSERT offer_price_history не найден"
-    assert "offer_price_history_listing_change_uq" in oph_sqls[0], (
+    assert (
-        "ON CONFLICT должен ссылаться на UNIQUE constraint"
+        "offer_price_history_listing_change_uq" in oph_sqls[0]
-    )
+    ), "ON CONFLICT должен ссылаться на UNIQUE constraint"
 
 
 def test_save_detail_enrichment_skips_price_change_without_change_time():
@@ -271,7 +280,7 @@ def test_save_detail_enrichment_skips_price_change_without_change_time():
     db = _mock_db_detail(price_rub=5_000_000)
     enrichment = DetailEnrichment(
         price_changes=[
-            {"change_time": None, "price_rub": 5_000_000},       # пропустить
+            {"change_time": None, "price_rub": 5_000_000},  # пропустить
             {"change_time": "2026-04-01T00:00:00Z", "price_rub": None},  # пропустить
             {"change_time": "2026-04-01T00:00:00Z", "price_rub": 5_000_000},  # записать
         ]

tradein-mvp/backend/tests/test_yandex_city_sweep.py

@@ -11,7 +11,9 @@ Part E1) — эквивалентная regression-coverage теперь в
 
 Coverage:
   - YandexCitySweepCounters корректно заполняются (dataclass shape)
-  - Combos watchdog timeout существенно больше ANCHOR_TIMEOUT_SEC (arithmetic-only)
+  - Combos watchdog timeout существенно больше ANCHOR_TIMEOUT_SEC (arithmetic-only;
+    `DEFAULT_PRICE_RANGES`/`ROOM_PATH` — kit `scraper_kit.providers.yandex.serp`,
+    ретаргетировано с удалённого `app.services.scrapers.yandex_realty`, #2397 Part E2)
   - Admin endpoints (start/cancel/list-runs) — kit sweep вызван с DI
     (config/matcher/enrichment)
 """
@@ -194,8 +196,7 @@ def test_combos_sweep_timeout_substantially_larger_than_anchor_timeout() -> None
         _YANDEX_COMBOS_PER_FETCH_S,
         ANCHOR_TIMEOUT_SEC,
     )
-
+    from scraper_kit.providers.yandex.serp import DEFAULT_PRICE_RANGES, ROOM_PATH
-    from app.services.scrapers.yandex_realty import DEFAULT_PRICE_RANGES, ROOM_PATH
 
     # Параметры prod-режима
     rooms_list = list(ROOM_PATH.keys())  # 5 комнатностей

tradein-mvp/backend/tests/test_yandex_scrapers_delay_wiring.py

@@ -1,8 +1,10 @@
 """Smoke: each Yandex scraper instance reads request_delay_sec from scraper_settings.
 
 Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
-финальный шаг scraper_kit-миграции). `yandex_realty` / `yandex_newbuilding` пока
+финальный шаг scraper_kit-миграции). `yandex_newbuilding` пока остаётся легаси (не
-остаются легаси (не мигрированы) — их delay-wiring тесты не тронуты.
+мигрирован) — его delay-wiring тест не тронут. `yandex_realty` удалён (#2397 Part
+E2, 0 runtime-импортёров) — его delay-wiring тест убран (kit `YandexRealtyScraper`
+delay-provider wiring покрыт `test_kit_serp_proxy_pool.py`).
 `yandex_detail`/`yandex_valuation` переведены на kit-эквиваленты, которые используют
 инжектируемый `delay_provider=` (strangler DI, #2133) вместо прямого импорта
 `get_scraper_delay` — тест адаптирован под реальный контракт kit-класса.
@@ -15,7 +17,6 @@ from scraper_kit.providers.yandex.detail import YandexDetailScraper
 from scraper_kit.providers.yandex.valuation import YandexValuationScraper
 
 from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
-from app.services.scrapers.yandex_realty import YandexRealtyScraper
 
 _KIT_CONFIG = SimpleNamespace(
     scraper_proxy_url=None,
@@ -24,12 +25,6 @@ _KIT_CONFIG = SimpleNamespace(
 )
 
 
-def test_yandex_realty_picks_up_delay():
-    with patch("app.services.scrapers.yandex_realty.get_scraper_delay", return_value=9.5):
-        s = YandexRealtyScraper()
-        assert s.request_delay_sec == 9.5
-
-
 def test_yandex_detail_picks_up_delay():
     """Kit YandexDetailScraper: delay_provider callable вызывается с self.name."""
     s = YandexDetailScraper(delay_provider=lambda _name: 9.5)

tradein-mvp/scripts/local-sweep-ekb-cian.py

@@ -1,359 +0,0 @@
-"""Cian ЕКБ vtorichka sweep — local residential IP → prod tradein DB через SSH tunnel.
-
-Why: ekb.cian.ru SERP отдаёт ВЕСЬ город (без geo-фильтра по lat/lon), поэтому
-"10 anchors" из Avito-эталона тут бесполезны — одинаковый ответ × 10.
-Вместо anchors — proper room-segmented citywide pagination
-(rooms 1/2/3/4 × N pages × ~28 listings), per-page save (visible progress +
-не теряем данные при крэше).
-
-Cian реально менее агрессивен чем Avito, но **residential IP всё равно может попасть
-в blacklist при слишком частых запросах** — default delay 15s.
-
-Setup (один раз):
-  ssh -f -N -L 15433:172.20.0.2:5432 gendesign
-  cd tradein-mvp/backend
-  $env:DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
-  $env:ENVIRONMENT='local'; $env:GLITCHTIP_DSN=''
-
-Usage:
-  uv run python ../scripts/local-sweep-ekb-cian.py                 # default: 4 rooms × 8 pages
-  uv run python ../scripts/local-sweep-ekb-cian.py --pages 20      # больше глубины
-  uv run python ../scripts/local-sweep-ekb-cian.py --delay 10      # быстрее (risk)
-  uv run python ../scripts/local-sweep-ekb-cian.py --rooms 0       # всё (включая студии — no room param)
-"""
-from __future__ import annotations
-
-# stdlib
-import argparse
-import asyncio
-import json
-import sys
-import time
-import types
-from pathlib import Path
-
-# Auto-inject backend/ в sys.path чтобы найти app.* package
-_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
-if str(_BACKEND_DIR) not in sys.path:
-    sys.path.insert(0, str(_BACKEND_DIR))
-
-# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
-for _m in (
-    "weasyprint",
-    "weasyprint.text",
-    "weasyprint.text.ffi",
-    "weasyprint.css",
-    "weasyprint.css.targets",
-):
-    sys.modules[_m] = types.ModuleType(_m)
-sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
-sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
-
-# app imports после stubs
-import logging  # noqa: E402
-import re  # noqa: E402
-
-from app.core.db import SessionLocal  # noqa: E402
-from app.services.scrapers.base import ScrapedLot, save_listings  # noqa: E402
-from app.services.scrapers.cian import CianScraper  # noqa: E402
-
-# Шумный WARNING из upstream cian_state_parser ('state extraction failed') — у нас
-# fallback на новый .concat() markup отрабатывает, warning нерелевантен.
-logging.getLogger("app.services.scrapers.cian").setLevel(logging.ERROR)
-
-# ── New-markup fallback parser ───────────────────────────────────────────────
-# Cian переключил MFE bootstrap с .push({...}) на .concat([{...},{...}]):
-#   window._cianConfig['frontend-serp'] = (window._cianConfig['frontend-serp'] || []).concat([…])
-# Существующий cian_state_parser.py не ловит этот шаблон → 0 lots на свежей выдаче.
-# Чтобы не трогать shared scraper module (правилами запрещено), парсим тут.
-_RE_CONCAT_START = re.compile(
-    r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]\s*=\s*"
-    r"\(window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\)\.concat\(\["
-)
-
-
-def _extract_balanced_array(text: str, start: int) -> str | None:
-    """Из позиции `start` (где '[') вернуть текст массива включая внешние '[' и ']'.
-
-    JSON-aware: учитывает strings и escapes, чтобы не сбить счётчик '[' ']' на
-    скобках внутри string literals.
-    """
-    if text[start] != "[":
-        return None
-    depth = 0
-    i = start
-    in_str = False
-    esc = False
-    while i < len(text):
-        ch = text[i]
-        if in_str:
-            if esc:
-                esc = False
-            elif ch == "\\":
-                esc = True
-            elif ch == '"':
-                in_str = False
-        else:
-            if ch == '"':
-                in_str = True
-            elif ch == "[":
-                depth += 1
-            elif ch == "]":
-                depth -= 1
-                if depth == 0:
-                    return text[start : i + 1]
-        i += 1
-    return None
-
-
-def _parse_serp_v2(scraper: CianScraper, html: str) -> list[ScrapedLot]:
-    """Fallback парсер для нового шаблона .concat([...]) — ekb.cian.ru/cat.php c 2026-05."""
-    state: dict | None = None
-    for m in _RE_CONCAT_START.finditer(html):
-        if m.group("mfe") != "frontend-serp":
-            continue
-        # Позиция '[' — это последний символ матча минус 1 (regex кончается на '[')
-        arr_start = m.end() - 1
-        arr_str = _extract_balanced_array(html, arr_start)
-        if arr_str is None:
-            continue
-        try:
-            entries = json.loads(arr_str)
-        except json.JSONDecodeError:
-            continue
-        for entry in entries:
-            if (
-                isinstance(entry, dict)
-                and entry.get("key") == "initialState"
-                and isinstance(entry.get("value"), dict)
-            ):
-                state = entry["value"]
-                break
-        if state is not None:
-            break
-    if state is None:
-        return []
-    offers = (state.get("results") or {}).get("offers") or []
-    lots: list[ScrapedLot] = []
-    for offer in offers:
-        # Cian свежий markup: descriptionMinhash = list[int] (multi-hash signatures).
-        # ScrapedLot ожидает str → сбрасываем чтобы _offer_to_lot вычислил SHA1 от description.
-        if isinstance(offer.get("descriptionMinhash"), list):
-            offer["descriptionMinhash"] = None
-        lot = scraper._offer_to_lot(offer)
-        if lot is not None:
-            lots.append(lot)
-    return lots
-
-
-def _load_cookies_file(path: Path) -> dict[str, str]:
-    """Загрузить cookies из JSON-файла.
-
-    Поддерживает два формата:
-    - browser export (list[{name,value,...}]) — DevTools / EditThisCookie / Cookie-Editor
-    - plain dict {name: value}
-    """
-    raw = json.loads(path.read_text(encoding="utf-8"))
-    if isinstance(raw, list):
-        return {c["name"]: c["value"] for c in raw if c.get("name") and c.get("value")}
-    if isinstance(raw, dict):
-        return {str(k): str(v) for k, v in raw.items()}
-    raise SystemExit(f"--cookies-file: unknown JSON shape (got {type(raw).__name__})")
-
-
-async def phase_citywide(
-    pages: int,
-    delay: float,
-    room_segments: tuple[tuple[int, ...] | None, ...],
-    cookies_file: Path | None = None,
-    start_page: int = 1,
-) -> int:
-    """Room-segmented citywide pagination — per-page save, incremental.
-
-    room_segments — кортеж из tuple (1,) / (2,) / ... / None (= no room filter).
-    None segment даёт смешанную выдачу (студии + всё подряд) — Cian
-    не различает их без room param.
-    """
-    seg_labels = [
-        ("r" + str(seg[0]) if seg else "all")
-        for seg in room_segments
-    ]
-    print(
-        f"\n=== PHASE CITYWIDE: Cian ЕКБ vtorichka "
-        f"({len(room_segments)} segments={seg_labels} × {pages} pages, delay {delay}s) ===\n",
-        flush=True,
-    )
-    total = 0
-    cum_ins = 0
-    cum_upd = 0
-    async with CianScraper() as s:
-        # Override per-request delay (in addition to between-anchors delay)
-        s.request_delay_sec = delay
-        # Inject cookies (если есть): прокидываем в curl_cffi session перед warm-up
-        if cookies_file is not None:
-            cookies = _load_cookies_file(cookies_file)
-            assert s._cffi is not None
-            s._cffi.cookies.update(cookies)
-            print(f"  loaded {len(cookies)} cookies from {cookies_file.name}", flush=True)
-        # Warm-up: посетить homepage чтобы получить _CIAN_GK / _yasc + не выглядеть как наглый бот
-        try:
-            assert s._cffi is not None
-            r_warm = await s._cffi.get(f"{s.base_url}/")
-            print(
-                f"  warmup GET / → HTTP {r_warm.status_code} "
-                f"(cookies={len(list(s._cffi.cookies.jar))})",
-                flush=True,
-            )
-            # Subsequent SERP requests should look like in-site navigation
-            s._cffi.headers.update(
-                {
-                    "Referer": f"{s.base_url}/",
-                    "Sec-Fetch-Site": "same-origin",
-                    "Sec-Fetch-Mode": "navigate",
-                    "Sec-Fetch-User": "?1",
-                    "Sec-Fetch-Dest": "document",
-                }
-            )
-            await asyncio.sleep(min(5.0, delay))
-        except Exception as e:  # noqa: BLE001
-            print(f"  warmup FAIL: {type(e).__name__}: {e}", flush=True)
-        for seg_idx, seg in enumerate(room_segments):
-            seg_label = "r" + str(seg[0]) if seg else "all"
-            # --start-page применяется только к первому segment'у — продолжение прерванного sweep
-            page_from = start_page if seg_idx == 0 else 1
-            for page in range(page_from, pages + 1):
-                # Manual fetch+parse чтобы можно было дампнуть HTML при empty results
-                try:
-                    url_ = s._build_url(rooms=seg, page=page)
-                    assert s._cffi is not None
-                    resp = await s._cffi.get(url_)
-                    if resp.status_code != 200:
-                        print(
-                            f"  seg={seg_label} page={page} HTTP {resp.status_code} → skip",
-                            flush=True,
-                        )
-                        await asyncio.sleep(delay)
-                        continue
-                    lots = s._parse_serp_html(resp.text)
-                    if not lots:
-                        # Fallback: новый markup .concat([...])
-                        lots = _parse_serp_v2(s, resp.text)
-                        if lots:
-                            print(
-                                f"    (parsed via v2 .concat fallback)",
-                                flush=True,
-                            )
-                    if not lots:
-                        # Дамп для разбора: captcha vs пустая страница vs изменение markup
-                        dump_path = Path(f"cian-empty-seg-{seg_label}-p{page}.html")
-                        dump_path.write_text(resp.text, encoding="utf-8")
-                        print(f"    DUMP: {dump_path} ({len(resp.text)} bytes)", flush=True)
-                    await s.sleep_between_requests()
-                except Exception as e:  # noqa: BLE001
-                    print(
-                        f"  seg={seg_label} page={page} FAIL fetch: "
-                        f"{type(e).__name__}: {e}",
-                        flush=True,
-                    )
-                    await asyncio.sleep(delay)
-                    continue
-                if not lots:
-                    print(
-                        f"  seg={seg_label} page={page}: 0 lots — end of segment "
-                        "(либо captcha)",
-                        flush=True,
-                    )
-                    break
-                db = SessionLocal()
-                try:
-                    ins, upd = save_listings(db, lots)
-                    cum_ins += ins
-                    cum_upd += upd
-                    total += len(lots)
-                    print(
-                        f"  seg={seg_label} page={page}/{pages}: {len(lots)} lots → "
-                        f"ins={ins} upd={upd} "
-                        f"[cum: total={total} new={cum_ins} upd={cum_upd}]",
-                        flush=True,
-                    )
-                except Exception as e:  # noqa: BLE001
-                    print(
-                        f"  seg={seg_label} page={page} FAIL save: "
-                        f"{type(e).__name__}: {e}",
-                        flush=True,
-                    )
-                finally:
-                    db.close()
-                # scraper.fetch_around() уже спит request_delay_sec * jitter — extra sleep не нужен
-    print(
-        f"  PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}",
-        flush=True,
-    )
-    return total
-
-
-async def main() -> None:
-    p = argparse.ArgumentParser()
-    p.add_argument(
-        "--pages",
-        type=int,
-        default=8,
-        help="страниц на room-segment (default 8 → ~28×4×8 = ~900 до дедупа)",
-    )
-    p.add_argument(
-        "--delay",
-        type=float,
-        default=15,
-        help="sec между HTTP requests (residential-safe ≥10, default 15)",
-    )
-    p.add_argument(
-        "--rooms",
-        type=str,
-        default="1,2,3,4",
-        help="room segments через запятую (1,2,3,4) или 'all' для одного citywide "
-        "запроса без room param (default '1,2,3,4')",
-    )
-    p.add_argument(
-        "--cookies-file",
-        type=Path,
-        default=None,
-        help="JSON-файл с Cian cookies (browser-export или {name:value}). "
-        "Опционально — для обхода captcha. См. scripts/.cian-cookies.json",
-    )
-    p.add_argument(
-        "--start-page",
-        type=int,
-        default=1,
-        help="С какой страницы начать ПЕРВЫЙ сегмент (для resume прерванного sweep). "
-        "Остальные segments идут с p=1. Default 1.",
-    )
-    args = p.parse_args()
-
-    # Parse rooms argument
-    if args.rooms.strip().lower() == "all":
-        segments: tuple[tuple[int, ...] | None, ...] = (None,)
-    else:
-        try:
-            nums = [int(x.strip()) for x in args.rooms.split(",") if x.strip()]
-        except ValueError as e:
-            raise SystemExit(f"--rooms must be comma-separated ints or 'all': {e}") from e
-        segments = tuple((n,) for n in nums)
-
-    cookies_file: Path | None = args.cookies_file
-    if cookies_file is not None and not cookies_file.exists():
-        raise SystemExit(f"--cookies-file not found: {cookies_file}")
-
-    t0 = time.perf_counter()
-    await phase_citywide(
-        args.pages,
-        args.delay,
-        segments,
-        cookies_file=cookies_file,
-        start_page=args.start_page,
-    )
-    elapsed = time.perf_counter() - t0
-    print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

tradein-mvp/scripts/local-sweep-ekb-yandex.py

@@ -1,376 +0,0 @@
-"""Yandex.Недвижимость ЕКБ vtorichka sweep — runs locally через residential IP.
-
-Why local: Yandex anti-bot (captcha redirect) включается быстрее на datacenter
-IP + httpx fingerprint. Запуск с residential IP + Chrome120 TLS impersonation
-(curl_cffi) + browser session cookies — проходит. Скрипт пишет в prod tradein
-DB через SSH tunnel (localhost:15433 → tradein-postgres docker container).
-
-Modes:
-  --mode citywide  (default): один общий /vtorichniy-rynok/ sweep, ~25 страниц
-                              × 23 cards = ~575 уникальных. Yandex SERP лимит.
-  --mode combos:    итерация 5 rooms × 6 price ranges = 30 combos × до 25 pages.
-                    DB-level дедуп → coverage близко к total (~4000+ vtorichka
-                    ЕКБ). ~75 min полный прогон.
-
-Setup:
-  ssh -f -N -L 15433:172.20.0.2:5432 gendesign   # tradein-postgres tunnel
-  cd tradein-mvp/backend
-  export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
-  export ENVIRONMENT='local'
-  export GLITCHTIP_DSN=''
-
-Usage:
-  uv run python ../scripts/local-sweep-ekb-yandex.py                       # citywide
-  uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos          # full ~4000
-  uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos \\
-      --rooms 1,2 --price-ranges 5-7,7-10                                   # subset
-  uv run python ../scripts/local-sweep-ekb-yandex.py --pages 1              # smoke
-"""
-from __future__ import annotations
-
-# stdlib
-import argparse
-import asyncio
-import json
-import random
-import sys
-import time
-import types
-from pathlib import Path
-from urllib.parse import urlencode
-
-# Auto-inject backend/ в sys.path чтобы найти app.* package
-_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
-if str(_BACKEND_DIR) not in sys.path:
-    sys.path.insert(0, str(_BACKEND_DIR))
-
-# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
-for _m in (
-    "weasyprint",
-    "weasyprint.text",
-    "weasyprint.text.ffi",
-    "weasyprint.css",
-    "weasyprint.css.targets",
-):
-    sys.modules[_m] = types.ModuleType(_m)
-sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
-sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
-
-# app imports после stubs
-from curl_cffi.requests import AsyncSession  # noqa: E402
-
-from app.core.db import SessionLocal  # noqa: E402
-from app.services.scrapers.base import save_listings  # noqa: E402
-from app.services.scrapers.yandex_realty import YandexRealtyScraper  # noqa: E402
-
-DEFAULT_CITY = "ekaterinburg"
-DEFAULT_COOKIES_FILE = Path(__file__).resolve().parent / ".yandex-cookies.json"
-
-# Path-сегменты Yandex для room buckets (после канонизации). Ключ — slug для
-# --rooms CLI; значение — segment в URL.
-ROOM_PATH: dict[str, str] = {
-    "studio": "studiya",
-    "1": "odnokomnatnaya",
-    "2": "dvuhkomnatnaya",
-    "3": "trehkomnatnaya",
-    "4+": "4-i-bolee",
-}
-
-# Price ranges (price_min, price_max) в рублях. None — open-ended boundary.
-PRICE_RANGES: list[tuple[int | None, int | None]] = [
-    (None, 5_000_000),
-    (5_000_000, 7_000_000),
-    (7_000_000, 10_000_000),
-    (10_000_000, 15_000_000),
-    (15_000_000, 25_000_000),
-    (25_000_000, None),
-]
-
-_CAPTCHA_MARKERS = ("/showcaptcha", "smartcaptcha", "checkboxcaptcha")
-
-
-def _load_cookies(path: Path) -> dict[str, str]:
-    if not path.exists():
-        return {}
-    raw = json.loads(path.read_text(encoding="utf-8"))
-    return {item["name"]: item["value"] for item in raw if "name" in item and "value" in item}
-
-
-def _is_captcha(html: str) -> bool:
-    head = html[:5000].lower()
-    return any(m in head for m in _CAPTCHA_MARKERS) or "докажите, что вы не робот" in head
-
-
-def _build_url(
-    city: str,
-    page: int = 0,
-    rooms: str | None = None,
-    price_min: int | None = None,
-    price_max: int | None = None,
-) -> str:
-    """Yandex SERP URL. Если rooms задан — path-based бакет; иначе общий vtorichka.
-
-    newFlat=NO_DEAL принудительно отфильтровывает новостройки (нужно при
-    combo, т.к. Yandex canonicalize иногда дропает /vtorichniy-rynok/).
-    """
-    if rooms:
-        path = f"/{city}/kupit/kvartira/{ROOM_PATH[rooms]}/vtorichniy-rynok/"
-    else:
-        path = f"/{city}/kupit/kvartira/vtorichniy-rynok/"
-    base = f"https://realty.yandex.ru{path}"
-
-    params: dict[str, str | int] = {}
-    if price_min is not None:
-        params["priceMin"] = price_min
-    if price_max is not None:
-        params["priceMax"] = price_max
-    if price_min is not None or price_max is not None:
-        # Гарантия vtorichka — Yandex иногда дропает /vtorichniy-rynok/ при canonicalize
-        params["newFlat"] = "NO_DEAL"
-    if page > 0:
-        params["page"] = page
-
-    if params:
-        return f"{base}?{urlencode(params)}"
-    return base
-
-
-def _combo_label(rooms: str | None, price_min: int | None, price_max: int | None) -> str:
-    r = rooms or "all-rooms"
-    if price_min is None and price_max is None:
-        return f"{r}/any-price"
-    lo = f"{price_min // 1_000_000}M" if price_min else "0"
-    hi = f"{price_max // 1_000_000}M" if price_max else "inf"
-    return f"{r}/{lo}-{hi}"
-
-
-async def sweep_one(
-    session: AsyncSession,
-    parser: YandexRealtyScraper,
-    city: str,
-    pages: int,
-    delay: float,
-    rooms: str | None = None,
-    price_min: int | None = None,
-    price_max: int | None = None,
-) -> tuple[int, int, int]:
-    """Один scan по фиксированному (rooms, price) — пагинация до pages или empty.
-
-    Returns: (fetched, inserted, updated)
-    """
-    label = _combo_label(rooms, price_min, price_max)
-    total = cum_ins = cum_upd = 0
-    for page in range(pages):
-        url = _build_url(city, page=page, rooms=rooms, price_min=price_min, price_max=price_max)
-        try:
-            resp = await session.get(url, timeout=20)
-        except Exception as e:  # noqa: BLE001
-            print(f"    [{label}] page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
-            await asyncio.sleep(delay)
-            continue
-        if resp.status_code != 200:
-            print(f"    [{label}] page={page} HTTP {resp.status_code} — stop combo", flush=True)
-            break
-        if _is_captcha(resp.text):
-            print(
-                f"    [{label}] page={page} ⚠️ CAPTCHA — stop combo. "
-                f"Подожди 30-60min или refresh .yandex-cookies.json",
-                flush=True,
-            )
-            raise RuntimeError("captcha")
-        # NBSP → space: yandex_helpers.parse_rub() багует на \xa0 в group(1) → price None
-        html = resp.text.replace("\xa0", " ")
-        lots = parser._parse_html(html, page=page)
-        if not lots:
-            print(f"    [{label}] page={page}: 0 lots — end of pagination, stop", flush=True)
-            break
-        db = SessionLocal()
-        try:
-            ins, upd = save_listings(db, lots)
-            cum_ins += ins
-            cum_upd += upd
-            total += len(lots)
-            print(
-                f"    [{label}] page={page + 1}/{pages}: {len(lots)} lots → "
-                f"ins={ins} upd={upd} [combo: {total}/{cum_ins}/{cum_upd}]",
-                flush=True,
-            )
-        except Exception as e:  # noqa: BLE001
-            print(f"    [{label}] page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
-        finally:
-            db.close()
-        if page + 1 < pages:
-            await asyncio.sleep(delay * random.uniform(0.85, 1.3))
-    return total, cum_ins, cum_upd
-
-
-async def phase_citywide(pages: int, delay: float, city: str, cookies: dict[str, str]) -> None:
-    print(
-        f"\n=== PHASE CITYWIDE: Yandex vtorichka (city={city}, {pages} pages, "
-        f"delay {delay}s) ===\n",
-        flush=True,
-    )
-    parser = YandexRealtyScraper(city=city)
-    async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
-        total, ins, upd = await sweep_one(session, parser, city, pages, delay)
-    print(f"  CITYWIDE TOTAL: fetched={total} ins={ins} upd={upd}", flush=True)
-
-
-async def phase_combos(
-    pages: int,
-    delay: float,
-    city: str,
-    cookies: dict[str, str],
-    rooms_list: list[str],
-    price_ranges: list[tuple[int | None, int | None]],
-    combo_delay: float,
-) -> None:
-    combos = [(r, lo, hi) for r in rooms_list for lo, hi in price_ranges]
-    print(
-        f"\n=== PHASE COMBOS: {len(combos)} combos × ≤{pages} pages "
-        f"(city={city}, delay {delay}s, combo-delay {combo_delay}s) ===",
-        flush=True,
-    )
-    parser = YandexRealtyScraper(city=city)
-    grand_fetched = grand_ins = grand_upd = 0
-    async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
-        for i, (rooms, lo, hi) in enumerate(combos, 1):
-            label = _combo_label(rooms, lo, hi)
-            print(f"\n  [{i}/{len(combos)}] combo={label}", flush=True)
-            try:
-                f, ins, upd = await sweep_one(
-                    session, parser, city, pages, delay,
-                    rooms=rooms, price_min=lo, price_max=hi,
-                )
-            except RuntimeError as e:
-                if str(e) == "captcha":
-                    print("  ⚠️ stopping combo run — captcha", flush=True)
-                    break
-                raise
-            grand_fetched += f
-            grand_ins += ins
-            grand_upd += upd
-            print(
-                f"  [{i}/{len(combos)}] {label} done: f={f} i={ins} u={upd} "
-                f"[grand: f={grand_fetched} i={grand_ins} u={grand_upd}]",
-                flush=True,
-            )
-            if i < len(combos):
-                await asyncio.sleep(combo_delay * random.uniform(0.85, 1.3))
-    print(
-        f"\n  COMBOS GRAND TOTAL: fetched={grand_fetched} "
-        f"ins={grand_ins} upd={grand_upd}",
-        flush=True,
-    )
-
-
-def _parse_rooms_arg(s: str) -> list[str]:
-    if s == "all":
-        return list(ROOM_PATH)
-    out: list[str] = []
-    for tok in s.split(","):
-        tok = tok.strip()
-        if tok not in ROOM_PATH:
-            raise argparse.ArgumentTypeError(
-                f"unknown room '{tok}'. Allowed: {sorted(ROOM_PATH)} or 'all'"
-            )
-        out.append(tok)
-    return out
-
-
-def _parse_price_ranges_arg(s: str) -> list[tuple[int | None, int | None]]:
-    if s == "all":
-        return list(PRICE_RANGES)
-    out: list[tuple[int | None, int | None]] = []
-    for tok in s.split(","):
-        tok = tok.strip()
-        if "-" not in tok:
-            raise argparse.ArgumentTypeError(f"range '{tok}' missing '-'")
-        lo_s, hi_s = tok.split("-", 1)
-        lo = int(float(lo_s) * 1_000_000) if lo_s else None
-        hi = int(float(hi_s) * 1_000_000) if hi_s else None
-        out.append((lo, hi))
-    return out
-
-
-async def main() -> None:
-    p = argparse.ArgumentParser(description="Yandex.Недвижимость ЕКБ sweep")
-    p.add_argument(
-        "--mode",
-        choices=["citywide", "combos"],
-        default="citywide",
-        help="citywide = один общий vtorichka sweep (~575 уникальных). "
-        "combos = rooms × price = ~4000+",
-    )
-    p.add_argument(
-        "--pages",
-        type=int,
-        default=25,
-        help="max pages per sweep (Yandex SERP cap ≈ 25; stops earlier on empty)",
-    )
-    p.add_argument(
-        "--delay",
-        type=float,
-        default=15,
-        help="sec между HTTP requests внутри combo (default 15, residential-safe)",
-    )
-    p.add_argument(
-        "--combo-delay",
-        type=float,
-        default=20,
-        help="sec между combos (default 20, дополнительный jitter combos mode)",
-    )
-    p.add_argument(
-        "--rooms",
-        type=_parse_rooms_arg,
-        default="all",
-        help=f"combos: comma-list из {sorted(ROOM_PATH)} или 'all' (default)",
-    )
-    p.add_argument(
-        "--price-ranges",
-        type=_parse_price_ranges_arg,
-        default="all",
-        help="combos: comma-list 'lo-hi' в млн руб ('-5,5-7,7-10,10-15,15-25,25-') "
-        "или 'all' (default). Empty side = open-ended.",
-    )
-    p.add_argument(
-        "--city",
-        type=str,
-        default=DEFAULT_CITY,
-        help=f"Yandex city slug (default '{DEFAULT_CITY}')",
-    )
-    p.add_argument(
-        "--cookies-file",
-        type=Path,
-        default=DEFAULT_COOKIES_FILE,
-        help=f"Browser cookies JSON (default '{DEFAULT_COOKIES_FILE.name}'). "
-        f"Empty/missing → start без cookies (риск captcha).",
-    )
-    args = p.parse_args()
-
-    cookies = _load_cookies(args.cookies_file)
-    if cookies:
-        print(f"loaded {len(cookies)} cookies from {args.cookies_file}", flush=True)
-    else:
-        print(
-            f"⚠️ no cookies loaded ({args.cookies_file}, exists={args.cookies_file.exists()}). "
-            f"Yandex anti-bot risk.",
-            flush=True,
-        )
-
-    t0 = time.perf_counter()
-    if args.mode == "citywide":
-        await phase_citywide(args.pages, args.delay, args.city, cookies)
-    else:
-        await phase_combos(
-            args.pages, args.delay, args.city, cookies,
-            rooms_list=args.rooms,
-            price_ranges=args.price_ranges,
-            combo_delay=args.combo_delay,
-        )
-    elapsed = time.perf_counter() - t0
-    print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())

tradein-mvp/scripts/local-sweep-ekb.py

@@ -1,443 +0,0 @@
-"""Full ЕКБ sweep + houses enrichment — runs locally через residential IP.
-
-Why: prod VPS Beget IP заблокирован Avito (datacenter fingerprint → 429).
-Local IP (домашний ISP) = residential → проходит. Скрипт пишет в prod DB
-через SSH tunnel (localhost:15433 → tradein-postgres docker container).
-
-Setup:
-  ssh -f -N -L 15433:172.20.0.2:5432 gendesign   # tradein-postgres tunnel
-  cd tradein-mvp/backend
-  export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
-
-Usage:
-  uv run python ../scripts/local-sweep-ekb.py                     # defaults
-  uv run python ../scripts/local-sweep-ekb.py --pages 3 --detail-top 20 --delay 30
-  uv run python ../scripts/local-sweep-ekb.py --skip-search       # только detail + houses
-  uv run python ../scripts/local-sweep-ekb.py --skip-houses       # без houses
-"""
-from __future__ import annotations
-
-# stdlib
-import argparse
-import asyncio
-import sys
-import time
-import types
-from pathlib import Path
-from urllib.parse import urlparse
-
-# Auto-inject backend/ в sys.path чтобы найти app.* package
-# (скрипт лежит в tradein-mvp/scripts/, app — в tradein-mvp/backend/)
-_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
-if str(_BACKEND_DIR) not in sys.path:
-    sys.path.insert(0, str(_BACKEND_DIR))
-
-# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
-for _m in (
-    "weasyprint",
-    "weasyprint.text",
-    "weasyprint.text.ffi",
-    "weasyprint.css",
-    "weasyprint.css.targets",
-):
-    sys.modules[_m] = types.ModuleType(_m)
-sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
-sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})
-
-# app imports после stubs
-from sqlalchemy import text  # noqa: E402
-
-from app.core.db import SessionLocal  # noqa: E402
-from app.services.scrapers.avito import AvitoScraper  # noqa: E402
-
-
-class CityWideAvitoScraper(AvitoScraper):
-    """Subclass без geo-фильтра — search всего города (ЕКБ) через pagination.
-
-    Override `_build_web_url` чтобы НЕ добавлять `geoCoords`/`radius` — Avito
-    отдаёт ВСЕ объявления в городе sorted by date, до 100 pages (cap ~5000).
-    """
-
-    def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str:  # noqa: ARG002
-        from urllib.parse import urlencode
-        params = {"s": 104, "p": page}  # s=104 sort by date
-        return f"{self.base_url}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode(params)}"
-
-
-# Avito URL slugs для room-filter (extracted из real search page HTML 2026-05-23)
-ROOM_SLUGS: list[tuple[str, str]] = [
-    ("студии",       "studii-ASgBAgICAkSSA8YQygj~WA"),
-    ("1-комн.",      "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"),
-    ("2-комн.",      "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"),
-    ("3-комн.",      "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"),
-    ("4-комн.",      "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"),
-    ("5+комн.",      "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"),
-    ("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"),
-]
-
-
-class CityRoomsAvitoScraper(AvitoScraper):
-    """Subclass с URL slug filter по комнатности — для splitting когда total > 5000.
-
-    URL pattern: /ekaterinburg/kvartiry/prodam/<slug>?s=104&p=N
-    """
-
-    def __init__(self, slug: str) -> None:
-        super().__init__()
-        self.room_slug = slug
-
-    def _build_web_url(self, lat: float, lon: float, radius_km: int, page: int = 1) -> str:  # noqa: ARG002
-        from urllib.parse import urlencode
-        params = {"s": 104, "p": page}
-        return f"{self.base_url}/ekaterinburg/kvartiry/prodam/{self.room_slug}?{urlencode(params)}"
-from app.services.scrapers.avito_detail import (  # noqa: E402
-    fetch_detail,
-    save_detail_enrichment,
-)
-from app.services.scrapers.avito_houses import (  # noqa: E402
-    fetch_house_catalog,
-    save_house_catalog_enrichment,
-)
-from app.services.scrapers.base import save_listings  # noqa: E402
-
-# 10 anchors покрывающих центральные/северные/южные/восточные/западные районы ЕКБ
-ANCHORS: list[tuple[str, float, float]] = [
-    ("Центр",         56.8400, 60.6050),
-    ("ЮЗ",            56.7950, 60.5300),
-    ("Уралмаш",       56.8970, 60.6100),
-    ("Академический", 56.7700, 60.5500),
-    ("Пионерский",    56.8650, 60.6200),
-    ("Эльмаш",        56.8567, 60.6700),
-    ("Втузгородок",   56.8400, 60.6650),
-    ("ВИЗ",           56.8500, 60.5650),
-    ("Юго-Запад",     56.7770, 60.5800),
-    ("Сортировка",    56.8950, 60.5600),
-]
-
-
-async def phase_citywide(pages: int, delay: float) -> int:
-    """City-wide pagination — ВСЕ объявления ЕКБ (no geo filter), per-page save.
-
-    Per-page save важно: visible прогресс + не теряем данные если crash mid-way.
-    """
-    print(
-        f"\n=== PHASE CITYWIDE: full ЕКБ pagination ({pages} pages, delay {delay}s) ===\n",
-        flush=True,
-    )
-    total = 0
-    cum_ins = 0
-    cum_upd = 0
-    async with CityWideAvitoScraper() as s:
-        for page in range(1, pages + 1):
-            url = s._build_web_url(0.0, 0.0, 1, page=page)
-            try:
-                assert s._cffi is not None
-                response = await s._cffi.get(url)
-                if response.status_code != 200:
-                    print(f"  ⚠️ page={page} HTTP {response.status_code} — stop", flush=True)
-                    break
-                lots = s._parse_html(response.text, source_url_base=url)
-            except Exception as e:  # noqa: BLE001
-                print(f"  page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
-                await asyncio.sleep(delay)
-                continue
-            if not lots:
-                print(f"  page={page}: 0 lots — end of pagination", flush=True)
-                break
-            db = SessionLocal()
-            try:
-                ins, upd = save_listings(db, lots)
-                cum_ins += ins
-                cum_upd += upd
-                total += len(lots)
-                print(
-                    f"  page={page}/{pages}: {len(lots)} lots → ins={ins} upd={upd} "
-                    f"[cum: total={total} new={cum_ins} upd={cum_upd}]",
-                    flush=True,
-                )
-            except Exception as e:  # noqa: BLE001
-                print(f"  page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
-            finally:
-                db.close()
-            if page < pages:
-                await asyncio.sleep(delay)
-    print(f"  PHASE CITYWIDE TOTAL: fetched={total} ins={cum_ins} upd={cum_upd}", flush=True)
-    return total
-
-
-async def phase_byrooms(pages: int, delay: float) -> int:
-    """Rooms-split sweep — 7 categories × pages, per-page save.
-
-    Для total > 5000 (Avito search cap) split по комнатности позволяет получить
-    все listings (каждый segment <5000).
-    """
-    print(
-        f"\n=== PHASE BYROOMS: 7 категорий × {pages} pages, delay {delay}s ===\n",
-        flush=True,
-    )
-    grand_total = 0
-    grand_ins = 0
-    grand_upd = 0
-    import random
-    rate_limit_hits = 0
-    for ci, (name, slug) in enumerate(ROOM_SLUGS, 1):
-        print(f"\n--- [{ci}/{len(ROOM_SLUGS)}] Категория: {name} (slug={slug[:30]}...) ---", flush=True)
-        cat_total = 0
-        cat_ins = 0
-        cat_upd = 0
-        async with CityRoomsAvitoScraper(slug) as s:
-            for page in range(1, pages + 1):
-                url = s._build_web_url(0.0, 0.0, 1, page=page)
-                try:
-                    assert s._cffi is not None
-                    response = await s._cffi.get(url)
-                    if response.status_code == 429:
-                        # Exponential backoff на rate-limit: 60s × 2^N (max 30 min)
-                        rate_limit_hits += 1
-                        backoff = min(60 * (2 ** (rate_limit_hits - 1)), 1800)
-                        print(f"  {name} p={page} HTTP 429 (hit #{rate_limit_hits}) → backoff {backoff}s", flush=True)
-                        await asyncio.sleep(backoff)
-                        # retry same page after backoff
-                        try:
-                            response = await s._cffi.get(url)
-                            if response.status_code != 200:
-                                print(f"  {name} p={page} retry HTTP {response.status_code} — skip page", flush=True)
-                                await asyncio.sleep(delay * 2)
-                                continue
-                        except Exception as e:  # noqa: BLE001
-                            print(f"  {name} p={page} retry FAIL: {type(e).__name__}: {e}", flush=True)
-                            continue
-                    elif response.status_code != 200:
-                        print(f"  {name} p={page} HTTP {response.status_code} — skip page", flush=True)
-                        await asyncio.sleep(delay)
-                        continue
-                    lots = s._parse_html(response.text, source_url_base=url)
-                except Exception as e:  # noqa: BLE001
-                    print(f"  {name} p={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
-                    await asyncio.sleep(delay)
-                    continue
-                if not lots:
-                    print(f"  {name} page={page}: 0 lots — end of category", flush=True)
-                    break
-                db = SessionLocal()
-                try:
-                    ins, upd = save_listings(db, lots)
-                    cat_ins += ins
-                    cat_upd += upd
-                    cat_total += len(lots)
-                    print(
-                        f"  {name} p={page}/{pages}: {len(lots)} lots → ins={ins} upd={upd} "
-                        f"[cat: tot={cat_total} new={cat_ins}]",
-                        flush=True,
-                    )
-                except Exception as e:  # noqa: BLE001
-                    print(f"  {name} page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
-                finally:
-                    db.close()
-                if page < pages:
-                    # Jitter: ±30% от delay чтобы выглядеть менее robotic
-                    jittered = delay * random.uniform(0.7, 1.3)
-                    await asyncio.sleep(jittered)
-        print(
-            f"  --- {name} TOTAL: fetched={cat_total} ins={cat_ins} upd={cat_upd} ---",
-            flush=True,
-        )
-        grand_total += cat_total
-        grand_ins += cat_ins
-        grand_upd += cat_upd
-    print(
-        f"\n  PHASE BYROOMS GRAND TOTAL: fetched={grand_total} new={grand_ins} updated={grand_upd}",
-        flush=True,
-    )
-    return grand_total
-
-
-async def phase_a_search(pages: int, delay: float) -> int:
-    """Search всех anchors × pages, save listings."""
-    print(
-        f"\n=== PHASE A: search ({len(ANCHORS)} anchors × {pages} pages, "
-        f"delay {delay}s) ===\n",
-        flush=True,
-    )
-    total = 0
-    for i, (name, lat, lon) in enumerate(ANCHORS, 1):
-        print(f"[{i}/{len(ANCHORS)}] {name} ({lat},{lon})", flush=True)
-        try:
-            async with AvitoScraper() as s:
-                lots = await s.fetch_around(
-                    lat, lon, 1500, pages=pages, delay_override_sec=delay
-                )
-        except Exception as e:  # noqa: BLE001
-            print(f"  FAIL fetch: {type(e).__name__}: {e}", flush=True)
-            await asyncio.sleep(delay)
-            continue
-        print(f"  fetched {len(lots)} lots", flush=True)
-        if not lots:
-            print("  ⚠️ 0 lots — possible 429, stop phase A", flush=True)
-            break
-        db = SessionLocal()
-        try:
-            ins, upd = save_listings(db, lots)
-            print(f"  saved: ins={ins} upd={upd}", flush=True)
-            total += len(lots)
-        except Exception as e:  # noqa: BLE001
-            print(f"  FAIL save: {type(e).__name__}: {e}", flush=True)
-        finally:
-            db.close()
-        if i < len(ANCHORS):
-            await asyncio.sleep(delay)
-    return total
-
-
-async def phase_b_detail(top_n: int, delay: float) -> tuple[int, set[str]]:
-    """Detail enrichment для top-N recent listings без detail.
-
-    Returns: (enriched_count, set of unique house_catalog_url paths extracted)
-    """
-    print(
-        f"\n=== PHASE B: detail enrichment (top {top_n}, delay {delay}s) ===\n",
-        flush=True,
-    )
-    db = SessionLocal()
-    try:
-        rows = db.execute(
-            text(
-                """
-                SELECT source_url, source_id FROM listings
-                WHERE source='avito' AND detail_enriched_at IS NULL
-                  AND source_url IS NOT NULL AND price_rub > 0
-                ORDER BY scraped_at DESC NULLS LAST
-                LIMIT :limit
-                """
-            ),
-            {"limit": top_n},
-        ).mappings().all()
-    finally:
-        db.close()
-
-    print(f"  queue: {len(rows)} listings to enrich", flush=True)
-    enriched = 0
-    failed = 0
-    house_urls: set[str] = set()
-    for i, row in enumerate(rows, 1):
-        url = row["source_url"]
-        path = urlparse(url).path if url.startswith("http") else url
-        print(f"[{i}/{len(rows)}] detail {row['source_id']}", flush=True)
-        try:
-            enr = await fetch_detail(path)
-            db = SessionLocal()
-            try:
-                ok = save_detail_enrichment(db, enr)
-                print(
-                    f"  enriched: ok={ok} "
-                    f"rooms={enr.rooms} area={enr.area_m2} "
-                    f"kitchen={enr.kitchen_area_m2} owners={enr.owners_count} "
-                    f"house_url={enr.house_catalog_url}",
-                    flush=True,
-                )
-                if ok:
-                    enriched += 1
-                # Извлекаем house path (strip query string + filter только /catalog/houses/)
-                if enr.house_catalog_url:
-                    p = urlparse(enr.house_catalog_url).path
-                    if p.startswith("/catalog/houses/"):
-                        house_urls.add(p)
-            finally:
-                db.close()
-        except Exception as e:  # noqa: BLE001
-            print(f"  FAIL: {type(e).__name__}: {e}", flush=True)
-            failed += 1
-        if i < len(rows):
-            await asyncio.sleep(delay)
-    print(f"  total: enriched={enriched} failed={failed} unique_houses_extracted={len(house_urls)}", flush=True)
-    return enriched, house_urls
-
-
-async def phase_c_houses(delay: float, extra_urls: set[str] | None = None) -> int:
-    """Houses enrichment — unique house_url из listings (DB) + extra_urls (из Phase B in-memory).
-
-    save_detail_enrichment не пишет house_url в listings (баг Stage 2b),
-    поэтому Phase C принимает extra_urls собранные Phase B вручную.
-    """
-    print("\n=== PHASE C: houses enrichment ===\n", flush=True)
-    db = SessionLocal()
-    try:
-        rows = db.execute(
-            text(
-                """
-                SELECT DISTINCT house_url FROM listings
-                WHERE source='avito' AND house_url IS NOT NULL
-                  AND house_id_fk IS NULL
-                ORDER BY house_url
-                """
-            )
-        ).fetchall()
-    finally:
-        db.close()
-
-    urls_from_db = {r[0] for r in rows if r[0]}
-    urls = urls_from_db | (extra_urls or set())
-    print(f"  queue: {len(urls)} unique houses (db={len(urls_from_db)} +extras={len(extra_urls or set())})", flush=True)
-    if not urls:
-        print("  no houses to enrich (skip)", flush=True)
-        return 0
-
-    enriched = 0
-    failed = 0
-    for i, url in enumerate(sorted(urls), 1):
-        path = urlparse(url).path if url.startswith("http") else url
-        print(f"[{i}/{len(urls)}] {path}", flush=True)
-        try:
-            enr = await fetch_house_catalog(path)
-            db = SessionLocal()
-            try:
-                counters = save_house_catalog_enrichment(db, enr)
-                print(f"  saved: {counters}", flush=True)
-                enriched += 1
-            finally:
-                db.close()
-        except Exception as e:  # noqa: BLE001
-            print(f"  FAIL: {type(e).__name__}: {e}", flush=True)
-            failed += 1
-        if i < len(urls):
-            await asyncio.sleep(delay)
-    print(f"  total: enriched={enriched} failed={failed}", flush=True)
-    return enriched
-
-
-async def main() -> None:
-    p = argparse.ArgumentParser()
-    p.add_argument(
-        "--mode",
-        choices=["anchors", "citywide", "byrooms", "both"],
-        default="anchors",
-        help="anchors=10 anchors; citywide=full pagination без geo; byrooms=7 категорий комнат (для total>5000); both=anchors+citywide",
-    )
-    p.add_argument("--pages", type=int, default=5, help="pages per anchor (anchors mode) или total pages (citywide)")
-    p.add_argument("--detail-top", type=int, default=50, help="top-N для detail enrichment")
-    p.add_argument("--delay", type=float, default=30, help="sec между HTTP requests")
-    p.add_argument("--skip-search", action="store_true")
-    p.add_argument("--skip-detail", action="store_true")
-    p.add_argument("--skip-houses", action="store_true")
-    args = p.parse_args()
-
-    t0 = time.perf_counter()
-    if not args.skip_search:
-        if args.mode in ("anchors", "both"):
-            await phase_a_search(args.pages, args.delay)
-        if args.mode in ("citywide", "both"):
-            cw_pages = args.pages if args.mode == "citywide" else 100
-            await phase_citywide(cw_pages, args.delay)
-        if args.mode == "byrooms":
-            await phase_byrooms(args.pages, args.delay)
-    extracted_house_urls: set[str] = set()
-    if not args.skip_detail:
-        _, extracted_house_urls = await phase_b_detail(args.detail_top, args.delay)
-    if not args.skip_houses:
-        await phase_c_houses(args.delay, extra_urls=extracted_house_urls)
-    elapsed = time.perf_counter() - t0
-    print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)
-
-
-if __name__ == "__main__":
-    asyncio.run(main())