From eef23071c5d21dba4422cca2901f6b93d3e7e170 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 27 Jun 2026 16:39:16 +0300 Subject: [PATCH 1/8] =?UTF-8?q?feat(tradein/domclick):=20Layer=20B=20detai?= =?UTF-8?q?l-backfill=20(SSR=20parse=20+=20AVM)=20=E2=80=94=20DORMANT/park?= =?UTF-8?q?ed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ПАРКОВКА (не мержить до прокси): код готов — parser на реальных __SSR_STATE__ путях (productCard.objectInfo/priceInfo.priceHistory/egrnData/houseInfo.info + pricePrediction AVM), orchestrator зеркало avito_detail_backfill, schedule 139 enabled=false. Деплоить нельзя: DomClick card/offer-card за QRATOR reputation-403 на прод mobile-прокси (нужен dedicated residential RU IP). Детали/аудит транспорта — Forgejo issue. --- tradein-mvp/backend/app/services/scheduler.py | 52 ++ .../app/services/scrapers/domclick_detail.py | 584 ++++++++++++++++++ .../services/scrapers/domclick_exceptions.py | 14 + .../app/tasks/domclick_detail_backfill.py | 272 ++++++++ .../sql/139_seed_domclick_detail_backfill.sql | 64 ++ .../tests/scrapers/test_domclick_detail.py | 404 ++++++++++++ .../tests/test_domclick_detail_backfill.py | 195 ++++++ 7 files changed, 1585 insertions(+) create mode 100644 tradein-mvp/backend/app/services/scrapers/domclick_detail.py create mode 100644 tradein-mvp/backend/app/tasks/domclick_detail_backfill.py create mode 100644 tradein-mvp/backend/data/sql/139_seed_domclick_detail_backfill.sql create mode 100644 tradein-mvp/backend/tests/scrapers/test_domclick_detail.py create mode 100644 tradein-mvp/backend/tests/test_domclick_detail_backfill.py diff --git a/tradein-mvp/backend/app/services/scheduler.py b/tradein-mvp/backend/app/services/scheduler.py index 281369f5..41b65629 100644 --- a/tradein-mvp/backend/app/services/scheduler.py +++ b/tradein-mvp/backend/app/services/scheduler.py @@ -89,6 +89,17 @@ Sources: city_sweep; fetch via curl_cffi chrome120 + scraper_proxy_url, parse via YandexDetailScraper.parse; window 12:00-15:00 UTC — после avito_detail_backfill 09-12 UTC) + - domclick_detail_backfill → run_domclick_detail_backfill + (tasks/domclick_detail_backfill.py, Layer B #1846; nightly backfill + of domclick (source='domklik') detail_enriched_at. Layer A SERP sweep + collects only base fields; the card page (behind DataDome → + BrowserFetcher transport ONLY, curl_cffi → 401) carries + renovation/area/balconies/encumbrances/priceHistory. ONE + BrowserFetcher session per run; block-abort → mark_done (transient), + parse-failure → rollback+retry next run. Shipped DORMANT (seed 139, + enabled=false) — enable after prod-smoke; depends on Layer A having + populated source='domklik' listings. window 15:00-18:00 UTC — после + avito (09-12) и yandex (12-15) detail backfills) - cadastral_geo_match → run_cadastral_geo_match (tasks/cadastral_geo_match.py; combined nightly job — (1) REFRESH cad_buildings_local from gendesign_cad_buildings FDW (one bulk @@ -1554,6 +1565,45 @@ async def trigger_yandex_detail_backfill_run( return run_id +async def trigger_domclick_detail_backfill_run( + db: Session, schedule_row: dict[str, Any] +) -> int | None: + """Create scrape_run + launch run_domclick_detail_backfill in asyncio.create_task. + + Layer B detail-enrichment for legacy domclick listings (source='domklik'). Layer A + (domclick_city_sweep / #1979) collects only base SERP fields; the card page (behind + DataDome → BrowserFetcher only, curl_cffi → 401) carries renovation/repair_state, + living/kitchen area, balconies, sale_type, owners_count, encumbrances, year_built, + views, priceHistory. + + Mirrors trigger_yandex_detail_backfill_run: claim run -> create_task -> + mark_done/failed delegated to tasks/domclick_detail_backfill.run_domclick_detail_backfill. + + Returns run_id or None (skip -- already running). + """ + run_id = _claim_run(db, schedule_row) + if run_id is None: + return None + + params = schedule_row.get("default_params") or {} + + async def _run() -> None: + run_db = SessionLocal() + try: + from app.tasks.domclick_detail_backfill import run_domclick_detail_backfill + + await run_domclick_detail_backfill(run_db, run_id=run_id, params=params) + except Exception: + logger.exception("scheduler: run_domclick_detail_backfill crashed run_id=%d", run_id) + finally: + run_db.close() + + task = asyncio.create_task(_run()) + task.add_done_callback(lambda t: t.exception() if not t.cancelled() else None) + logger.info("scheduler: triggered domclick_detail_backfill run_id=%d", run_id) + return run_id + + async def trigger_cadastral_geo_match_run(db: Session, schedule_row: dict[str, Any]) -> int | None: """Создать scrape_runs + launch run_cadastral_geo_match в executor (sync DB-only task). @@ -1795,6 +1845,8 @@ async def scheduler_loop() -> None: await trigger_avito_detail_backfill_run(db, sch) elif source == "yandex_detail_backfill": await trigger_yandex_detail_backfill_run(db, sch) + elif source == "domclick_detail_backfill": + await trigger_domclick_detail_backfill_run(db, sch) elif source == "cadastral_geo_match": await trigger_cadastral_geo_match_run(db, sch) elif source == "house_imv_backfill": diff --git a/tradein-mvp/backend/app/services/scrapers/domclick_detail.py b/tradein-mvp/backend/app/services/scrapers/domclick_detail.py new file mode 100644 index 00000000..041df810 --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/domclick_detail.py @@ -0,0 +1,584 @@ +"""DomClick.ru Layer B — detail-card enrichment (#1846 follow-up). + +Layer A (domclick.py) собирает базовые поля через BFF JSON API. Layer B дочитывает +страницу карточки (``https://ekaterinburg.domclick.ru/card/sale__flat__``) и +обогащает listings: renovation/repair_state, living/kitchen area, balconies, +sale_type, owners_count, encumbrances, year_built, views и priceHistory. + +Транспорт — BrowserFetcher через QRATOR-clean прокси (source="cian"): card-хосты +DomClick заблокированы QRATOR через domclick/avito-прокси, но чисты через cian +(verified live 2026-06-27). curl_cffi на card-хосте отдаёт 401/стаб. fetch_detail +принимает уже открытый browser_fetcher; жизненным циклом сессии и выбором source +владеет оркестратор. + +SSR-quirk +--------- +HTML карточки встраивает ``window.__SSR_STATE__ = {...}`` — это JS object literal, +а НЕ чистый JSON: в value-позиции встречается bare ``undefined`` (json.loads падает). +_extract_ssr_state делает balanced-brace scan (с пропуском скобок внутри строк) и +заменяет bare ``undefined`` → ``null`` строго в value-позиции, не трогая строки. + +Форма SSR-стейта подтверждена на живой карточке 2075729321 (2026-06-27): +productCard.objectInfo.{renovation,livingArea,kitchenArea}, productCard.priceInfo. +priceHistory ({date ISO8601+tz, price, diff, state}), productCard.egrnData +(snake_case owners_count/collateral/collateral_sber), productCard.legalOptions. +saleType, productCard.viewsCount/callsCount, houseInfo.info.{wallType,floorType}, +и ТОП-УРОВНЕМ pricePrediction (DomClick AVM → raw_extra.avm). +Schedule для Layer B ships DORMANT — включается вручную после prod-smoke. +""" + +from __future__ import annotations + +import json +import logging +import re +from dataclasses import dataclass, field +from datetime import UTC, datetime +from typing import TYPE_CHECKING, Any + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services.scrapers.domclick_exceptions import DomClickBlockedError, DomClickParseError +from app.services.scrapers.repair_state_normalizer import ( + infer_repair_state_from_text, + normalize_repair_state, +) + +if TYPE_CHECKING: + from app.services.scrapers.browser_fetcher import BrowserFetcher + +logger = logging.getLogger(__name__) + +# ── Маркеры anti-bot challenge (DataDome / QRATOR) ─────────────────────────── +# Зеркало идеи Layer A: если __SSR_STATE__ отсутствует И страница похожа на +# challenge — это блок, а не parse-failure. +_BLOCK_MARKERS: tuple[str, ...] = ("qrator", "captcha", "datadome", "access denied") + +# ── item_id из URL карточки: .../card/sale__flat__2075729321 ───────────────── +_ITEM_ID_RE = re.compile(r"sale__flat__(\d+)") + +# ── SSR assignment marker ──────────────────────────────────────────────────── +_SSR_ASSIGN_RE = re.compile(r"window\.__SSR_STATE__\s*=\s*") + +# ── bare `undefined` (value-позиция) → null, не трогая строки ───────────────── +_UNDEFINED_RE = re.compile(r"(?<=[:\[,\s])undefined(?=[\s,\]\}])") + +# ── renovation (RU) → каноничный repair_state enum ─────────────────────────── +# ЛОКАЛЬНАЯ карта (не трогаем shared _RAW_TO_ENUM в repair_state_normalizer). +# Значения уже каноничны; прогоняются через normalize_repair_state для валидации. +_RENOVATION_MAP: dict[str, str] = { + "евроремонт": "good", + "косметический": "standard", + "дизайнерский": "excellent", + "черновая": "needs_repair", + "без отделки": "needs_repair", + "требует ремонта": "needs_repair", + "предчистовая": "needs_repair", +} + + +# ── DomClickDetailEnrichment ────────────────────────────────────────────────── +@dataclass +class DomClickDetailEnrichment: + """Результат парсинга detail-карточки DomClick (Layer B). + + save_detail_enrichment() пишет подмножество в listings (+ offer_price_history). + Все поля None-safe: отсутствующее → None (не крэш). + """ + + item_id: str | None = None + source_url: str | None = None + + repair_state: str | None = None # enum: needs_repair/standard/good/excellent + repair_type: str | None = None # сырое RU-значение renovation + living_area_m2: float | None = None + kitchen_area_m2: float | None = None + balconies_count: int | None = None + has_balcony: bool | None = None + sale_type: str | None = None + owners_count: int | None = None + encumbrances_clean: bool | None = None # True=чисто, False=есть обременение + year_built: int | None = None + views_total: int | None = None + + price_changes: list[dict[str, Any]] = field(default_factory=list) + raw_extra: dict[str, Any] = field(default_factory=dict) + + +# ── SSR state extraction ────────────────────────────────────────────────────── + + +def _scan_balanced_object(s: str, start: int) -> str: + """Сканирует от '{' по индексу ``start`` до парной '}'. + + Отслеживает глубину по {/}, ИГНОРИРУЯ скобки внутри JS-строк (одинарные и + двойные кавычки с ``\\``-эскейпами). Возвращает литерал ``{...}`` включительно. + + Raises: + DomClickParseError: если парная скобка не найдена (truncated HTML). + """ + depth = 0 + in_string = False + quote_char = "" + escaped = False + for i in range(start, len(s)): + ch = s[i] + if in_string: + if escaped: + escaped = False + elif ch == "\\": + escaped = True + elif ch == quote_char: + in_string = False + continue + if ch in ('"', "'"): + in_string = True + quote_char = ch + elif ch == "{": + depth += 1 + elif ch == "}": + depth -= 1 + if depth == 0: + return s[start : i + 1] + raise DomClickParseError("__SSR_STATE__ object not balanced (no matching brace)") + + +def _extract_ssr_state(html: str) -> dict[str, Any]: + """Извлекает ``window.__SSR_STATE__`` object-литерал из HTML карточки. + + Шаги: + 1. Находим присваивание ``window.__SSR_STATE__ =``. Если нет — различаем + challenge-страницу (→ DomClickBlockedError) и просто отсутствие + стейта (→ DomClickParseError). + 2. Balanced-brace scan от первого '{' → литерал. + 3. Санитайз bare ``undefined`` → ``null`` (только value-позиция). + 4. json.loads. + + Raises: + DomClickBlockedError: страница без стейта + маркеры anti-bot. + DomClickParseError: стейт не найден / не сбалансирован / json.loads упал. + """ + match = _SSR_ASSIGN_RE.search(html) + if match is None: + html_lower = html.lower() + if any(marker in html_lower for marker in _BLOCK_MARKERS): + raise DomClickBlockedError( + "DomClick detail: challenge page detected (no __SSR_STATE__, " + f"markers checked: {_BLOCK_MARKERS})" + ) + raise DomClickParseError("__SSR_STATE__ not found") + + brace_start = html.find("{", match.end()) + if brace_start == -1: + raise DomClickParseError("__SSR_STATE__ assignment has no opening '{'") + + literal = _scan_balanced_object(html, brace_start) + sanitized = _UNDEFINED_RE.sub("null", literal) + try: + data = json.loads(sanitized) + except json.JSONDecodeError as exc: + raise DomClickParseError( + f"__SSR_STATE__ json.loads failed: {exc} (head={sanitized[:200]!r})" + ) from exc + if not isinstance(data, dict): + raise DomClickParseError(f"__SSR_STATE__ is not an object: {type(data)}") + return data + + +# ── Value coercion helpers ──────────────────────────────────────────────────── + + +def _to_float(value: Any) -> float | None: + """None-safe float. bool/неконвертируемое → None.""" + if value is None or isinstance(value, bool): + return None + try: + return float(value) + except (ValueError, TypeError): + return None + + +def _to_int(value: Any) -> int | None: + """None-safe int. bool/неконвертируемое → None.""" + if value is None or isinstance(value, bool): + return None + try: + return int(value) + except (ValueError, TypeError): + return None + + +def _parse_change_time(value: Any) -> datetime | None: + """Нормализует дату изменения цены → timezone-aware datetime. + + Адаптация Layer A ``_parse_publish_date`` (которая возвращала date): здесь + поддерживаем epoch (сек/мс), числовую строку и ISO8601. Real priceHistory.date + приходит как ``2026-04-15T09:02:26.548728+03:00`` — fromisoformat СОХРАНЯЕТ + исходный offset (+03:00), НЕ конвертирует в UTC; naive-значения получают UTC. + epoch трактуем как UTC. Невалидное → None (caller пропускает запись). + """ + if value is None or isinstance(value, bool): + return None + # epoch (число): >1e12 трактуем как миллисекунды + if isinstance(value, int | float): + ts = float(value) + if ts > 1e12: + ts /= 1000.0 + try: + return datetime.fromtimestamp(ts, tz=UTC) + except (ValueError, OSError, OverflowError): + return None + s = str(value).strip() + if not s: + return None + if s.isdigit(): + ts = float(s) + if ts > 1e12: + ts /= 1000.0 + try: + return datetime.fromtimestamp(ts, tz=UTC) + except (ValueError, OSError, OverflowError): + return None + # ISO8601 (поддержим trailing 'Z') + try: + dt = datetime.fromisoformat(s.replace("Z", "+00:00")) + except (ValueError, TypeError): + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt + + +def _map_repair_state(renovation: str | None) -> str | None: + """RU renovation → каноничный enum. Нет совпадения в карте → инференс из текста.""" + if not renovation: + return None + mapped = _RENOVATION_MAP.get(renovation.strip().lower()) + if mapped is not None: + return normalize_repair_state(mapped) + return infer_repair_state_from_text(renovation) + + +def _compact(d: dict[str, Any]) -> dict[str, Any]: + """Выкидывает None-значения, чтобы raw_extra оставался компактным.""" + return {k: v for k, v in d.items() if v is not None} + + +def _pick(d: dict[str, Any], *keys: str) -> Any: + """Первое не-None значение среди ключей (defensive snake/camel spellings).""" + for k in keys: + v = d.get(k) + if v is not None: + return v + return None + + +# ── item_id ─────────────────────────────────────────────────────────────────── + + +def _extract_item_id(url: str) -> str | None: + """sale__flat__ из URL карточки.""" + m = _ITEM_ID_RE.search(url or "") + return m.group(1) if m else None + + +# ── parse_detail_html ────────────────────────────────────────────────────────── + + +def parse_detail_html(html: str, source_url: str) -> DomClickDetailEnrichment: + """Pure parser detail-карточки DomClick (без сети/БД). + + _extract_ssr_state может бросить DomClickBlockedError/DomClickParseError — + они propagate (оркестратор различает блок и parse-failure). Дальнейшая + навигация по полям обёрнута: per-field сюрприз формы → минимальный + enrichment (item_id+url), а не крэш. + """ + state = _extract_ssr_state(html) + item_id = _extract_item_id(source_url) + try: + pc = state.get("productCard") or {} + oi = pc.get("objectInfo") or {} + hi = (state.get("houseInfo") or {}).get("info") or {} + egrn = pc.get("egrnData") or {} + + # repair: enum + сырое значение + renovation = oi.get("renovation") + repair_type = renovation if isinstance(renovation, str) else None + repair_state = _map_repair_state(repair_type) + + # площади + living_area_m2 = _to_float(oi.get("livingArea")) + kitchen_area_m2 = _to_float(oi.get("kitchenArea")) + + # балконы: int → count+флаг; иное (строка/др.) → raw_extra, counts None + balconies = oi.get("balconies") + balconies_count: int | None = None + has_balcony: bool | None = None + balconies_raw: Any = None + if isinstance(balconies, bool): + balconies_raw = balconies + elif isinstance(balconies, int): + balconies_count = balconies + has_balcony = balconies > 0 + elif balconies is not None: + balconies_raw = balconies + + sale_type = (pc.get("legalOptions") or {}).get("saleType") + + # owners_count — защищаемся от двух написаний ключа + owners_count = _to_int(egrn.get("owners_count")) + if owners_count is None: + owners_count = _to_int(egrn.get("ownersCount")) + + # encumbrances_clean (ИНВЕРСИЯ): collateral/collateral_sber обозначают + # НАЛИЧИЕ обременения. Любой truthy → clean=False; оба отсутствуют/false + # при наличии egrnData → True; egrnData нет → None. + collateral = egrn.get("collateral") + collateral_sber = egrn.get("collateral_sber") + if egrn: + encumbrances_clean: bool | None = not bool(collateral or collateral_sber) + else: + encumbrances_clean = None + + year_built = _to_int(hi.get("buildYear")) + views_total = _to_int(pc.get("viewsCount")) + + # priceHistory → нормализованные записи (skip при битой дате / без цены) + raw_history = (pc.get("priceInfo") or {}).get("priceHistory") or [] + price_changes: list[dict[str, Any]] = [] + if isinstance(raw_history, list): + for entry in raw_history: + if not isinstance(entry, dict): + continue + change_time = _parse_change_time(entry.get("date")) + price_rub = _to_int(entry.get("price")) + # Skip при нераспарсенной дате ИЛИ отсутствующей/невалидной цене. + if change_time is None or price_rub is None: + continue + diff = entry.get("diff") + diff_percent = ( + diff if isinstance(diff, int | float) and not isinstance(diff, bool) else None + ) + price_changes.append( + { + "change_time": change_time, + "price_rub": price_rub, + "diff_percent": diff_percent, + } + ) + + # AVM (Layer C, тот же fetch) — DomClick price prediction живёт ТОП-УРОВНЕМ + # в SSR-стейте. Может быть плоским или обёрнут в .result/.data — defensively + # разворачиваем. В raw_extra пишем только присутствующие поля. + avm_root = state.get("pricePrediction") or {} + avm_src = avm_root + if isinstance(avm_root, dict): + avm_src = avm_root.get("result") or avm_root.get("data") or avm_root + if not isinstance(avm_src, dict): + avm_src = {} + avm = _compact( + { + "market_price": _pick(avm_src, "market_price", "marketPrice"), + "min": _pick(avm_src, "min_market_price", "minMarketPrice"), + "max": _pick(avm_src, "max_market_price", "maxMarketPrice"), + "repair_quality": _pick(avm_src, "repair_quality", "repairQuality"), + } + ) + + # raw_extra → merge в listings.raw_payload. wall/floor type ОСТАЮТСЯ тут, + # НЕ пишем в listings.house_type (кросс-источниковый словарь грязный). + # egrn area key — точное написание не подтверждено; пробуем несколько. + egrn_area = egrn.get("area") or egrn.get("rosreestrArea") or egrn.get("object_area") + demand = _compact( + { + "calls": pc.get("callsCount"), + "favorites": pc.get("favoriteOfferUsersCount"), + "duplicates": pc.get("duplicatesOfferCount"), + } + ) + raw_extra = _compact( + { + "wall_type": hi.get("wallType") or (pc.get("house") or {}).get("wallType"), + "floor_type": hi.get("floorType"), + "entrance_count": hi.get("entranceCount"), + "quarters_count": hi.get("quartersCount"), + "energy_efficiency": hi.get("energyEfficiency"), + "building_series": hi.get("buildingSeries"), + "domclick_building_guid": (pc.get("address") or {}).get("guid"), + "egrn_area": egrn_area, + "demand": demand or None, + "collateral": collateral, + "collateral_sber": collateral_sber, + "balconies_raw": balconies_raw, + "avm": avm or None, + } + ) + + return DomClickDetailEnrichment( + item_id=item_id, + source_url=source_url, + repair_state=repair_state, + repair_type=repair_type, + living_area_m2=living_area_m2, + kitchen_area_m2=kitchen_area_m2, + balconies_count=balconies_count, + has_balcony=has_balcony, + sale_type=sale_type if isinstance(sale_type, str) else None, + owners_count=owners_count, + encumbrances_clean=encumbrances_clean, + year_built=year_built, + views_total=views_total, + price_changes=price_changes, + raw_extra=raw_extra, + ) + except Exception: + logger.warning( + "domclick_detail: field navigation failed for %s — minimal enrichment", + source_url, + exc_info=True, + ) + return DomClickDetailEnrichment(item_id=item_id, source_url=source_url) + + +# ── fetch_detail ──────────────────────────────────────────────────────────────── + + +async def fetch_detail( + card_url: str, + *, + browser_fetcher: BrowserFetcher, +) -> DomClickDetailEnrichment: + """Fetch абсолютного URL карточки через BrowserFetcher → parse_detail_html. + + card_url уже абсолютный (listings.source_url из Layer A), URL не строим. + Сессией владеет оркестратор (передаёт уже открытый browser_fetcher). + + Raises: + DomClickBlockedError: anti-bot challenge ИЛИ ошибка браузерного fetch + (нет curl-fallback — DataDome даёт 401 на curl). Оркестратор считает + это блоком и помечает listing failed. + DomClickParseError: HTTP 200, но SSR-стейт не парсится (дрейф схемы). + """ + try: + html = await browser_fetcher.fetch(card_url) + except DomClickBlockedError: + raise + except Exception as exc: + raise DomClickBlockedError( + f"DomClick detail browser fetch failed for {card_url}: {exc}" + ) from exc + return parse_detail_html(html, card_url) + + +# ── save_detail_enrichment ────────────────────────────────────────────────────── + + +def save_detail_enrichment( + db: Session, + listing_id: int, + e: DomClickDetailEnrichment, +) -> bool: + """Persist enrichment в одной транзакции (commit в конце; rollback — за caller). + + 1. UPDATE listings SET (COALESCE — None не затирает существующее), + raw_payload || raw_extra, detail_enriched_at=NOW() WHERE id=listing_id. + 2. Для каждой записи priceHistory — INSERT INTO offer_price_history + (ON CONFLICT DO NOTHING → идемпотентно). source='domklik'. + + Snapshot (upsert_listing_snapshot) НЕ пишем: enrichment не несёт текущую цену, + а detail-backfill цену не меняет — snapshot остаётся за SERP-путём (note). + + Returns: + True если UPDATE задел строку (listing найден), иначе False. + """ + raw_extra_json = json.dumps(e.raw_extra or {}, ensure_ascii=False, default=str) + + result = db.execute( + text(""" + UPDATE listings SET + repair_state = COALESCE(:repair_state, repair_state), + repair_type = COALESCE(:repair_type, repair_type), + living_area_m2 = COALESCE(:living_area_m2, living_area_m2), + kitchen_area_m2 = COALESCE(:kitchen_area_m2, kitchen_area_m2), + balconies_count = COALESCE(:balconies_count, balconies_count), + has_balcony = COALESCE(:has_balcony, has_balcony), + sale_type = COALESCE(:sale_type, sale_type), + owners_count = COALESCE(:owners_count, owners_count), + encumbrances_clean = COALESCE(:encumbrances_clean, encumbrances_clean), + year_built = COALESCE(:year_built, year_built), + views_total = COALESCE(:views_total, views_total), + raw_payload = COALESCE(raw_payload, '{}'::jsonb) || CAST(:raw_extra AS jsonb), + detail_enriched_at = NOW() + WHERE id = CAST(:lid AS bigint) + """), + { + "lid": listing_id, + "repair_state": e.repair_state, + "repair_type": e.repair_type, + "living_area_m2": e.living_area_m2, + "kitchen_area_m2": e.kitchen_area_m2, + "balconies_count": e.balconies_count, + "has_balcony": e.has_balcony, + "sale_type": e.sale_type, + "owners_count": e.owners_count, + "encumbrances_clean": e.encumbrances_clean, + "year_built": e.year_built, + "views_total": e.views_total, + "raw_extra": raw_extra_json, + }, + ) + + # priceHistory — INSERT, де-дуп по (listing_id, change_time). SAVEPOINT per-row + # (backend.md): битый cast одной записи не валит весь UPDATE+commit. + for change in e.price_changes: + ct = change.get("change_time") + price = change.get("price_rub") + if not ct or not price: + continue + try: + with db.begin_nested(): + db.execute( + text(""" + INSERT INTO offer_price_history + (listing_id, change_time, price_rub, diff_percent, source) + VALUES ( + CAST(:lid AS bigint), + CAST(:ct AS timestamptz), + CAST(:price AS numeric), + CAST(:diff AS numeric), + 'domklik' + ) + ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq + DO NOTHING + """), + { + "lid": listing_id, + "ct": ct, + "price": price, + "diff": change.get("diff_percent"), + }, + ) + except Exception as exc: + logger.warning( + "domclick_detail: price-history insert failed listing_id=%s ct=%s: %s", + listing_id, + ct, + exc, + ) + + db.commit() + found = result.rowcount > 0 + if not found: + logger.warning( + "domclick_detail save: listing not found id=%s (item_id=%s)", + listing_id, + e.item_id, + ) + else: + logger.info( + "domclick_detail save: listing id=%s enriched (price_changes=%d)", + listing_id, + len(e.price_changes), + ) + return found diff --git a/tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py b/tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py index 796251cc..247de7fe 100644 --- a/tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py +++ b/tradein-mvp/backend/app/services/scrapers/domclick_exceptions.py @@ -11,3 +11,17 @@ class DomClickBlockedError(Exception): Обходится через shared mobile proxy (BrowserFetcher(source="domclick") → generic provider → мобильный egress). """ + + +class DomClickParseError(Exception): + """DomClick detail-карточка получена (HTTP 200, не block), но SSR-стейт + не извлекается / не парсится. + + Возникает в Layer B (domclick_detail.py) когда: + - `window.__SSR_STATE__` не найден в HTML (но это и не challenge-страница); + - object-литерал не сбалансирован по скобкам; + - json.loads падает после санитайза bare `undefined` → `null`. + + В отличие от DomClickBlockedError (anti-bot), это сигнал дрейфа схемы карточки + или повреждённого ответа — оркестратор считает его parse-failure, а не блоком. + """ diff --git a/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py b/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py new file mode 100644 index 00000000..313f4fff --- /dev/null +++ b/tradein-mvp/backend/app/tasks/domclick_detail_backfill.py @@ -0,0 +1,272 @@ +"""Scheduled backfill: DomClick Layer B detail-enrichment (#1846 follow-up). + +Nightly window 15:00-18:00 UTC (migration 139, source=domclick_detail_backfill). +Offset after avito (09-12) / yandex (12-15) detail backfills + the city sweeps to +avoid parallel egress on the shared mobile proxy. + +Problem: Layer A (domclick.py BFF sweep) собирает только базовые SERP-поля. Поля +карточки (renovation/repair_state, living/kitchen area, balconies, sale_type, +owners_count, encumbrances, year_built, views, priceHistory) живут только на +странице карточки (.../card/sale__flat__), за DataDome. + +Solution: single snapshot SELECT at start (guarantees termination). Транспорт — +BrowserFetcher(source="cian"): card-хосты DomClick заблокированы QRATOR через +domclick/avito-прокси, но чисты через cian-прокси (verified live 2026-06-27). +Одна браузер-сессия открывается на весь прогон и передаётся в fetch_detail. +Block-handling: DomClickBlockedError транзиентен (anti-bot / fetch-fail) +→ abort после max_consecutive_blocks с mark_done (НЕ mark_failed; retry следующей ночью +через NULL detail_enriched_at). DomClickParseError (HTTP 200, но SSR не парсится) → +листинг остаётся необогащённым (rollback), переотбирается на следующем прогоне. + +Schedule ships DORMANT (seed 139, enabled=false) — enable manually after prod-smoke. +Depends on Layer A (#1979 / domclick_city_sweep) having populated source='domklik' +listings first. +""" + +from __future__ import annotations + +import asyncio +import logging +import time +from dataclasses import dataclass, field + +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.services import scrape_runs as runs_mod +from app.services.scrapers.browser_fetcher import BrowserFetcher +from app.services.scrapers.domclick_detail import fetch_detail, save_detail_enrichment +from app.services.scrapers.domclick_exceptions import DomClickBlockedError, DomClickParseError + +logger = logging.getLogger(__name__) + +__all__ = [ + "DomClickDetailBackfillResult", + "run_domclick_detail_backfill", +] + + +@dataclass +class DomClickDetailBackfillResult: + """Counters for one domclick detail backfill run.""" + + attempted: int = 0 + enriched: int = 0 + blocked: int = 0 + failed: int = 0 + duration_sec: float = field(default=0.0) + + def to_dict(self) -> dict[str, int]: + return { + "attempted": self.attempted, + "enriched": self.enriched, + "blocked": self.blocked, + "failed": self.failed, + "duration_sec": int(self.duration_sec), + } + + +async def run_domclick_detail_backfill( + db: Session, + *, + run_id: int, + params: dict, +) -> DomClickDetailBackfillResult: + """Backfill detail_enriched_at for legacy domclick listings via BrowserFetcher. + + Params (from default_params jsonb in scrape_schedules): + batch_size: int -- snapshot size (SELECT LIMIT), default 300. + budget_sec: float -- wall-clock budget per run, default 3600s. + request_delay_sec: float -- delay between listings, default 8.0s. + max_consecutive_blocks: int -- consecutive blocks before abort, default 5. + + Transport: ONE BrowserFetcher(source="cian") for the whole run (DomClick card + hosts are QRATOR-clean only via the cian proxy). fetch_detail receives the + already-open fetcher. + + Lifecycle: update_heartbeat -> cancel-check -> snapshot -> loop with budget guard -> + mark_done (incl. partial / block-abort) / mark_failed (outer exception only). + """ + batch_size = int(params.get("batch_size", 300)) + budget_sec = float(params.get("budget_sec", 3600)) + request_delay_sec = float(params.get("request_delay_sec", 8.0)) + max_consecutive_blocks = int(params.get("max_consecutive_blocks", 5)) + + counters = DomClickDetailBackfillResult() + current_counters: dict[str, int] = counters.to_dict() + + start = time.monotonic() + + try: + runs_mod.update_heartbeat(db, run_id, current_counters) + + # Cooperative cancel before any browser launch / SELECT. The run is already in + # terminal 'cancelled' state (that is how is_cancelled detects it) — do NOT + # mark_done (would overwrite 'cancelled'), just return early. + if runs_mod.is_cancelled(db, run_id): + logger.info( + "domclick_detail_backfill: run_id=%d cancelled before loop -- returning early", + run_id, + ) + return counters + + # SNAPSHOT: single SELECT at start -- NOT re-selected in loop (termination guarantee). + snapshot = ( + db.execute( + text( + """ + SELECT id, source_id, source_url + FROM listings + WHERE source = 'domklik' + AND detail_enriched_at IS NULL + AND source_url IS NOT NULL + AND is_active = TRUE + ORDER BY scraped_at DESC NULLS LAST + LIMIT CAST(:batch_size AS int) + """ + ), + {"batch_size": batch_size}, + ) + .mappings() + .all() + ) + + if not snapshot: + logger.info( + "domclick_detail_backfill: run_id=%d -- no pending listings " + "(detail_enriched_at IS NULL = 0), done", + run_id, + ) + runs_mod.mark_done(db, run_id, current_counters) + return counters + + logger.info( + "domclick_detail_backfill: run_id=%d snapshot=%d (budget=%.0fs " + "delay=%.1fs max_blocks=%d)", + run_id, + len(snapshot), + budget_sec, + request_delay_sec, + max_consecutive_blocks, + ) + + consecutive_blocks = 0 + do_sleep = False + + # ONE browser session for the whole run (mirror Layer A domclick.py fetch_city). + # DomClick card/offer-card hosts are QRATOR-blocked via the domclick/avito + # proxies but clean via the cian proxy (verified live 2026-06-27); route detail + # fetches through it. + async with BrowserFetcher(source="cian") as bf: + for idx, row in enumerate(snapshot): + # Budget guard + elapsed = time.monotonic() - start + if elapsed > budget_sec: + logger.info( + "domclick_detail_backfill: run_id=%d -- budget %.0fs exhausted " + "(elapsed=%.1fs), stopping at #%d/%d", + run_id, + budget_sec, + elapsed, + idx, + len(snapshot), + ) + break + + # Delay before each request except the first + if do_sleep: + await asyncio.sleep(request_delay_sec) + do_sleep = True + + listing_id: int = row["id"] + source_url: str = row["source_url"] + counters.attempted += 1 + + try: + enrichment = await fetch_detail(source_url, browser_fetcher=bf) + if save_detail_enrichment(db, listing_id, enrichment): + counters.enriched += 1 + consecutive_blocks = 0 + + except DomClickBlockedError as exc: + # Anti-bot challenge / browser fetch failure. Transient — keep the + # inter-request delay (no IP rotation here, unlike avito) and abort + # the whole run after too many in a row (retried next night). + consecutive_blocks += 1 + counters.blocked += 1 + logger.warning( + "domclick_detail_backfill: run_id=%d BLOCKED #%d/%d (consecutive=%d): %s", + run_id, + idx + 1, + len(snapshot), + consecutive_blocks, + exc, + ) + if consecutive_blocks >= max_consecutive_blocks: + logger.error( + "domclick_detail_backfill: run_id=%d ABORT -- %d consecutive blocks " + "(anti-bot). enriched=%d attempted=%d", + run_id, + consecutive_blocks, + counters.enriched, + counters.attempted, + ) + break + + except DomClickParseError as exc: + # HTTP 200 but SSR unparseable (schema drift). Do NOT mark the listing + # enriched -> rollback so it is retried next run. + counters.failed += 1 + logger.warning( + "domclick_detail_backfill: run_id=%d listing_id=%d parse failure: %s", + run_id, + listing_id, + exc, + ) + try: + db.rollback() + except Exception: + pass + + except Exception as exc: + counters.failed += 1 + logger.warning( + "domclick_detail_backfill: run_id=%d listing_id=%d failed: %s", + run_id, + listing_id, + exc, + ) + try: + db.rollback() + except Exception: + pass + + if counters.attempted % 25 == 0: + current_counters = counters.to_dict() + runs_mod.update_heartbeat(db, run_id, current_counters) + + counters.duration_sec = time.monotonic() - start + current_counters = counters.to_dict() + runs_mod.update_heartbeat(db, run_id, current_counters) + runs_mod.mark_done(db, run_id, current_counters) + logger.info( + "domclick_detail_backfill: run_id=%d DONE -- attempted=%d enriched=%d " + "blocked=%d failed=%d duration=%.1fs", + run_id, + counters.attempted, + counters.enriched, + counters.blocked, + counters.failed, + counters.duration_sec, + ) + return counters + + except Exception as exc: + counters.duration_sec = time.monotonic() - start + logger.exception( + "domclick_detail_backfill: run_id=%d FAILED after %.1fs", + run_id, + counters.duration_sec, + ) + runs_mod.mark_failed(db, run_id, str(exc)[:1000], counters.to_dict()) + raise diff --git a/tradein-mvp/backend/data/sql/139_seed_domclick_detail_backfill.sql b/tradein-mvp/backend/data/sql/139_seed_domclick_detail_backfill.sql new file mode 100644 index 00000000..5415bcdc --- /dev/null +++ b/tradein-mvp/backend/data/sql/139_seed_domclick_detail_backfill.sql @@ -0,0 +1,64 @@ +-- 139_seed_domclick_detail_backfill.sql +-- Seed row для DomClick Layer B detail-backfill — периодическое обогащение карточек. +-- +-- !!! DORMANT BY DESIGN !!! enabled = false. +-- Layer A (domclick_city_sweep, #1979) собирает только базовые SERP-поля. Layer B +-- дочитывает страницу карточки (.../card/sale__flat__) и обогащает listings: +-- renovation/repair_state, living/kitchen area, balconies, sale_type, owners_count, +-- encumbrances, year_built, views, priceHistory. +-- +-- Транспорт — ТОЛЬКО BrowserFetcher(source="domclick"): карточки за DataDome, +-- curl_cffi отдаёт 401 (нет curl-fallback). Оркестратор открывает ОДНУ браузер-сессию +-- на весь прогон. Block (DataDome / fetch-fail) транзиентен → abort после +-- max_consecutive_blocks с mark_done (НЕ mark_failed); retry следующей ночью через +-- detail_enriched_at IS NULL. Parse-failure (HTTP 200, но SSR не парсится) → листинг +-- остаётся необогащённым (rollback) и переотбирается на следующем прогоне. +-- +-- Capability полностью wired (run_domclick_detail_backfill + trigger_domclick_detail_ +-- backfill_run + dispatch в scheduler.py), но schedule засеян ВЫКЛЮЧЕННЫМ — live-форма +-- SSR-стейта и поведение DataDome НЕ verified до ручного prod-smoke. ВКЛЮЧАТЬ ВРУЧНУЮ: +-- UPDATE scrape_schedules SET enabled = true WHERE source = 'domclick_detail_backfill'; +-- Зеркалит yandex_detail_backfill (113) / avito_detail_backfill (112) и dormant-сиды +-- domclick_city_sweep (122/138). +-- +-- ЗАВИСИМОСТИ: +-- - 052_scrape_schedules.sql (таблица + UNIQUE(source)). +-- - Layer A (#1979 / domclick_city_sweep) должен СНАЧАЛА наполнить listings +-- source='domklik' (иначе snapshot пуст → no-op mark_done на каждом прогоне). +-- Idempotent: ON CONFLICT (source) DO NOTHING — безопасно запускать повторно. +-- +-- Окно 15:00-18:00 UTC (18:00-21:00 МСК) — после avito (09-12) / yandex (12-15) detail +-- backfills и city sweeps, чтобы не конкурировать за egress на shared mobile proxy. +-- default_params (консервативно, под DataDome — browser-only транспорт медленный): +-- batch_size — размер snapshot (SELECT LIMIT) на один прогон. +-- budget_sec — wall-clock бюджет прогона (сек). +-- request_delay_sec — пауза между карточками (anti-DataDome). +-- max_consecutive_blocks — порог последовательных блоков до abort. + +BEGIN; + +-- next_run_at = следующее наступление окна (tomorrow + window_start_hour=15h). +-- При enabled=false get_due_schedules() строку не подхватит, но заполняем по образцу +-- 122/138: когда оператор включит schedule, next_run_at уже валиден и run не выстрелит +-- мгновенно (вне окна). AT TIME ZONE 'UTC' фиксирует wall-clock час в UTC независимо +-- от session TimeZone GUC. +INSERT INTO scrape_schedules ( + source, + enabled, + window_start_hour, + window_end_hour, + next_run_at, + default_params +) +VALUES +( + 'domclick_detail_backfill', + false, -- DORMANT — enable manually after prod-smoke + 15, + 18, + ((CURRENT_DATE + INTERVAL '1 day') + make_interval(hours => 15)) AT TIME ZONE 'UTC', + '{"batch_size": 300, "budget_sec": 3600, "request_delay_sec": 8, "max_consecutive_blocks": 5}'::jsonb +) +ON CONFLICT (source) DO NOTHING; + +COMMIT; diff --git a/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py b/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py new file mode 100644 index 00000000..3216ec29 --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_domclick_detail.py @@ -0,0 +1,404 @@ +"""Unit-тесты для DomClick Layer B detail-scraper (domclick_detail.py). + +Offline: parse/SSR-тесты без сети/БД; save-тест через MagicMock (зеркало +test_cian_detail.py — в проекте нет live-DB фикстуры, идемпотентность +гарантируется ON CONFLICT-клаузой, которая проверяется на уровне SQL-строки). + +Покрытие: +- _extract_ssr_state: bare `undefined` → null; in-string "undefined" выживает; + скобки/эскейпы внутри строк не ломают balanced-scan; missing marker → + DomClickParseError; challenge → DomClickBlockedError; unbalanced/невалидный + JSON → DomClickParseError. +- parse_detail_html (REAL __SSR_STATE__ paths, live card 2075729321): + productCard.objectInfo.{renovation→repair_state+repair_type,livingArea,kitchenArea}; + balconies absent→None; productCard.legalOptions.saleType; egrnData snake_case + owners_count + encumbrances инверсия (collateral); houseInfo.info.{wallType,floorType} + → raw_extra (НЕ house_type); productCard.viewsCount; priceInfo.priceHistory + (ISO8601+offset kept tz-aware / bad-date + no-price skipped); top-level + pricePrediction → raw_extra.avm (плоский + nested .result); raw_extra без None; + item_id из URL. +- fetch_detail: browser-branch + проброс ошибок. +- save_detail_enrichment: UPDATE-параметры, detail_enriched_at, price-history INSERT. +""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from app.services.scrapers.domclick_detail import ( + DomClickDetailEnrichment, + _extract_ssr_state, + fetch_detail, + parse_detail_html, + save_detail_enrichment, +) +from app.services.scrapers.domclick_exceptions import DomClickBlockedError, DomClickParseError + +_CARD_URL = "https://ekaterinburg.domclick.ru/card/sale__flat__2075729321" + +# Литерал с bare `undefined` (value-позиция), in-string "undefined", скобкой +# внутри строки и эскейп-кавычками — написан вручную, json.dumps использовать +# нельзя (нужен именно НЕвалидный-для-json undefined). +# Real __SSR_STATE__ shape (ground truth, live card 2075729321, 2026-06-27). +# bare `undefined` (value-pos), in-string "undefined", '{' внутри строки и +# эскейп-кавычки оставлены для проверки санитайзера/balanced-scan; json.dumps +# использовать нельзя (нужен именно НЕвалидный-для-json undefined). +# priceHistory[0].date намеренно flush-left (длинная ISO+offset строка) — отступ +# в литерале не влияет на парсер, но держит физическую строку ≤100 для ruff. +_SSR_LITERAL = """{ + "productCard": { + "objectInfo": { + "area": 38.0, + "rooms": 1, + "floor": 5, + "isApartment": false, + "renovation": "евроремонт", + "livingArea": 22.37, + "kitchenArea": 10.13, + "description": "Состояние пока undefined.", + "note": "угол {da}", + "quote": "сказал \\"привет\\" сосед" + }, + "legalOptions": {"saleType": "Свободная продажа"}, + "egrnData": { + "area": 38.2, + "floor": 5, + "owners_count": 1, + "collateral": true, + "collateral_sber": false + }, + "priceInfo": { + "priceHistory": [ +{"date": "2026-04-15T09:02:26.548728+03:00", "price": 13400000, "diff": -400000, "state": "less"}, + {"date": "not-a-date", "price": 4800000, "diff": -1.0, "state": "less"}, + {"date": "2026-05-01T10:00:00Z", "price": null, "diff": 0, "state": "more"} + ] + }, + "viewsCount": 338, + "callsCount": 5, + "favoriteOfferUsersCount": 12, + "duplicatesOfferCount": 0, + "address": {"guid": "abc-guid-123"}, + "unknownField": undefined + }, + "houseInfo": { + "info": { + "buildYear": 2015, + "wallType": "Кирпично-монолитный", + "floorType": "Железобетонный", + "entranceCount": 3, + "quartersCount": 120, + "energyEfficiency": "B", + "buildingSeries": "индивидуальный" + } + }, + "pricePrediction": { + "market_price": 13000000, + "min_market_price": 12000000, + "max_market_price": 14000000, + "rent_short": 50000, + "rent_long": 35000, + "repair_quality": "good" + } +}""" + +_HTML = ( + "" +) + + +def _wrap(state: dict) -> str: + """Заворачивает валидный dict в SSR-HTML (для variant-фикстур без undefined).""" + literal = json.dumps(state, ensure_ascii=False) + return f"" + + +# ── _extract_ssr_state ──────────────────────────────────────────────────────── + + +def test_extract_ssr_state_parses_despite_undefined() -> None: + state = _extract_ssr_state(_HTML) + # bare undefined в value-позиции → null → None + assert state["productCard"]["unknownField"] is None + # in-string "undefined" НЕ испорчен санитайзером + assert state["productCard"]["objectInfo"]["description"] == "Состояние пока undefined." + + +def test_extract_ssr_state_brace_and_escape_in_string() -> None: + state = _extract_ssr_state(_HTML) + oi = state["productCard"]["objectInfo"] + # '{' внутри строки не ломает balanced-brace scan + assert oi["note"] == "угол {da}" + # эскейп-кавычки внутри строки обработаны + assert oi["quote"] == 'сказал "привет" сосед' + + +def test_extract_ssr_state_missing_marker_raises_parse() -> None: + with pytest.raises(DomClickParseError): + _extract_ssr_state("no ssr state here") + + +def test_extract_ssr_state_challenge_raises_blocked() -> None: + html = "Access denied by DataDome captcha" + with pytest.raises(DomClickBlockedError): + _extract_ssr_state(html) + + +def test_extract_ssr_state_unbalanced_raises_parse() -> None: + with pytest.raises(DomClickParseError): + _extract_ssr_state('window.__SSR_STATE__ = {"a": 1') + + +def test_extract_ssr_state_invalid_json_raises_parse() -> None: + with pytest.raises(DomClickParseError): + _extract_ssr_state("window.__SSR_STATE__ = {this: is, not: json}") + + +# ── parse_detail_html — полная карточка ─────────────────────────────────────── + + +def test_parse_detail_html_full() -> None: + e = parse_detail_html(_HTML, _CARD_URL) + + assert e.item_id == "2075729321" + assert e.source_url == _CARD_URL + + assert e.repair_state == "good" # евроремонт → good (LOCAL map) + assert e.repair_type == "евроремонт" # сырое значение сохранено + + assert e.living_area_m2 == 22.37 + assert e.kitchen_area_m2 == 10.13 + + # На реальной карточке нет ключа balconies в objectInfo → optional/None. + assert e.balconies_count is None + assert e.has_balcony is None + + assert e.sale_type == "Свободная продажа" + assert e.owners_count == 1 + assert e.encumbrances_clean is False # collateral=true → есть обременение + assert e.year_built == 2015 + assert e.views_total == 338 + + +def test_parse_detail_html_price_changes() -> None: + e = parse_detail_html(_HTML, _CARD_URL) + # entry1 ок; entry2 bad-date skip; entry3 price=null skip → 1 запись + assert len(e.price_changes) == 1 + change = e.price_changes[0] + assert change["price_rub"] == 13400000 + assert change["diff_percent"] == -400000 + assert isinstance(change["change_time"], datetime) + # ISO8601-with-offset: tz-aware, offset СОХРАНЁН (+03:00), НЕ сконвертирован в UTC. + assert change["change_time"].tzinfo is not None + assert change["change_time"].utcoffset() == timedelta(hours=3) + + +def test_parse_detail_html_raw_extra() -> None: + e = parse_detail_html(_HTML, _CARD_URL) + raw = e.raw_extra + assert raw["wall_type"] == "Кирпично-монолитный" + assert raw["floor_type"] == "Железобетонный" + # wall/floor type живут ТОЛЬКО в raw_extra, НЕ в listings.house_type. + assert "house_type" not in raw + assert raw["domclick_building_guid"] == "abc-guid-123" + assert raw["egrn_area"] == 38.2 + assert raw["demand"]["calls"] == 5 + assert raw["demand"]["favorites"] == 12 + # AVM (Layer C, top-level pricePrediction) → raw_extra.avm + assert raw["avm"]["market_price"] == 13000000 + assert raw["avm"]["min"] == 12000000 + assert raw["avm"]["max"] == 14000000 + assert raw["avm"]["repair_quality"] == "good" + # нет ключа balconies → raw_extra НЕ содержит balconies_raw + assert "balconies_raw" not in raw + # никаких None-значений на верхнем уровне (компактность) + assert all(v is not None for v in raw.values()) + + +# ── parse_detail_html — частные ветки ───────────────────────────────────────── + + +def test_parse_balconies_string_goes_to_raw_extra() -> None: + html = _wrap({"productCard": {"objectInfo": {"balconies": "есть"}}}) + e = parse_detail_html(html, _CARD_URL) + assert e.balconies_count is None + assert e.has_balcony is None + assert e.raw_extra["balconies_raw"] == "есть" + + +def test_parse_encumbrances_clean_true_when_no_collateral() -> None: + html = _wrap({"productCard": {"egrnData": {"collateral": False, "collateral_sber": False}}}) + e = parse_detail_html(html, _CARD_URL) + assert e.encumbrances_clean is True + + +def test_parse_encumbrances_none_when_egrn_missing() -> None: + html = _wrap({"productCard": {"objectInfo": {"livingArea": 30}}}) + e = parse_detail_html(html, _CARD_URL) + assert e.encumbrances_clean is None + + +def test_parse_repair_map_hit() -> None: + html = _wrap({"productCard": {"objectInfo": {"renovation": "Косметический"}}}) + e = parse_detail_html(html, _CARD_URL) + assert e.repair_state == "standard" + assert e.repair_type == "Косметический" + + +def test_parse_repair_fallback_to_inference() -> None: + # Не точное совпадение в карте → инференс из текста (евроремонт → good) + html = _wrap({"productCard": {"objectInfo": {"renovation": "сделан свежий евроремонт"}}}) + e = parse_detail_html(html, _CARD_URL) + assert e.repair_state == "good" + assert e.repair_type == "сделан свежий евроремонт" + + +def test_parse_owners_count_camelcase_key() -> None: + html = _wrap({"productCard": {"egrnData": {"ownersCount": 3}}}) + e = parse_detail_html(html, _CARD_URL) + assert e.owners_count == 3 + + +def test_parse_avm_nested_under_result() -> None: + # pricePrediction может приходить обёрнутым в .result — unwrap должен сработать. + html = _wrap( + { + "productCard": {}, + "pricePrediction": {"result": {"marketPrice": 9000000, "repairQuality": "fine"}}, + } + ) + e = parse_detail_html(html, _CARD_URL) + assert e.raw_extra["avm"]["market_price"] == 9000000 + assert e.raw_extra["avm"]["repair_quality"] == "fine" + + +def test_parse_avm_absent_no_key() -> None: + # Нет pricePrediction → raw_extra без ключа avm (компактность). + e = parse_detail_html(_wrap({"productCard": {}}), _CARD_URL) + assert "avm" not in e.raw_extra + + +def test_parse_item_id_none_for_bad_url() -> None: + e = parse_detail_html(_wrap({"productCard": {}}), "https://domclick.ru/card/rent__room__1") + assert e.item_id is None + + +def test_dataclass_defaults() -> None: + e = DomClickDetailEnrichment() + assert e.item_id is None + assert e.price_changes == [] + assert e.raw_extra == {} + + +# ── fetch_detail (browser-branch) ───────────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_fetch_detail_parses_via_browser() -> None: + bf = MagicMock() + bf.fetch = AsyncMock(return_value=_HTML) + e = await fetch_detail(_CARD_URL, browser_fetcher=bf) + assert e.item_id == "2075729321" + assert e.repair_state == "good" + bf.fetch.assert_called_once_with(_CARD_URL) + + +@pytest.mark.asyncio +async def test_fetch_detail_browser_failure_raises_blocked() -> None: + bf = MagicMock() + bf.fetch = AsyncMock(side_effect=RuntimeError("502 bad gateway")) + with pytest.raises(DomClickBlockedError): + await fetch_detail(_CARD_URL, browser_fetcher=bf) + + +@pytest.mark.asyncio +async def test_fetch_detail_propagates_parse_error() -> None: + bf = MagicMock() + bf.fetch = AsyncMock(return_value="no state") + with pytest.raises(DomClickParseError): + await fetch_detail(_CARD_URL, browser_fetcher=bf) + + +@pytest.mark.asyncio +async def test_fetch_detail_propagates_blocked_from_html() -> None: + bf = MagicMock() + bf.fetch = AsyncMock(return_value="Access denied datadome") + with pytest.raises(DomClickBlockedError): + await fetch_detail(_CARD_URL, browser_fetcher=bf) + + +# ── save_detail_enrichment (MagicMock — зеркало test_cian_detail) ───────────── + + +def test_save_detail_enrichment_writes_columns() -> None: + db = MagicMock() + db.execute.return_value.rowcount = 1 + ct = datetime(2026, 4, 1, tzinfo=UTC) + e = DomClickDetailEnrichment( + item_id="2075729321", + source_url=_CARD_URL, + repair_state="good", + repair_type="Евроремонт", + living_area_m2=45.5, + kitchen_area_m2=12.3, + balconies_count=2, + has_balcony=True, + sale_type="свободная", + owners_count=1, + encumbrances_clean=False, + year_built=2015, + views_total=1234, + price_changes=[{"change_time": ct, "price_rub": 5000000, "diff_percent": -2.5}], + raw_extra={"wall_type": "монолит"}, + ) + + ok = save_detail_enrichment(db, 999, e) + assert ok is True + + # UPDATE — первый execute + update_sql = str(db.execute.call_args_list[0][0][0]) + update_params = db.execute.call_args_list[0][0][1] + assert "detail_enriched_at = NOW()" in update_sql + assert update_params["lid"] == 999 + assert update_params["repair_state"] == "good" + assert update_params["living_area_m2"] == 45.5 + assert update_params["balconies_count"] == 2 + assert update_params["has_balcony"] is True + assert update_params["encumbrances_clean"] is False + assert "монолит" in update_params["raw_extra"] + + # price-history INSERT с ON CONFLICT (идемпотентность) + insert_calls = [c for c in db.execute.call_args_list if "offer_price_history" in str(c[0][0])] + assert len(insert_calls) == 1 + assert "ON CONFLICT ON CONSTRAINT offer_price_history_listing_change_uq" in str( + insert_calls[0][0][0] + ) + ins_params = insert_calls[0][0][1] + assert ins_params["price"] == 5000000 + assert ins_params["ct"] == ct + db.commit.assert_called_once() + + +def test_save_detail_enrichment_listing_not_found() -> None: + db = MagicMock() + db.execute.return_value.rowcount = 0 + e = DomClickDetailEnrichment(item_id="x", source_url=_CARD_URL, repair_state="good") + assert save_detail_enrichment(db, 123, e) is False + + +def test_save_detail_enrichment_none_passthrough() -> None: + db = MagicMock() + db.execute.return_value.rowcount = 1 + e = DomClickDetailEnrichment(item_id="x", source_url=_CARD_URL) # все поля None + save_detail_enrichment(db, 1, e) + params = db.execute.call_args_list[0][0][1] + assert params["repair_state"] is None + assert params["living_area_m2"] is None + # пустой raw_extra → "{}" (merge no-op, не затирает) + assert params["raw_extra"] == "{}" diff --git a/tradein-mvp/backend/tests/test_domclick_detail_backfill.py b/tradein-mvp/backend/tests/test_domclick_detail_backfill.py new file mode 100644 index 00000000..c7fdca0a --- /dev/null +++ b/tradein-mvp/backend/tests/test_domclick_detail_backfill.py @@ -0,0 +1,195 @@ +"""Offline unit tests for run_domclick_detail_backfill (Layer B orchestrator). + +Pure & fast: NO live network, NO DB, NO browser. Mirrors test_domclick_sweep mocking +style. Mocks fetch_detail (AsyncMock) + save_detail_enrichment + scrape_runs lifecycle ++ a fake snapshot of listing rows via db.execute(...).mappings().all(). + +Coverage: + - happy path: both rows enriched -> mark_done(attempted=2, enriched=2) + - block-abort: DomClickBlockedError x max_consecutive_blocks -> abort, mark_done (not failed) + - parse-failure: DomClickParseError for one row -> failed++, db.rollback, other still enriched + - cancel before loop -> returns early (no snapshot SELECT, no mark_done) +""" + +from __future__ import annotations + +import os + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost:5432/test") + +from typing import Any +from unittest.mock import AsyncMock, MagicMock + +import pytest + +from app.services.scrapers.domclick_exceptions import DomClickBlockedError, DomClickParseError +from app.tasks import domclick_detail_backfill as mod +from app.tasks.domclick_detail_backfill import run_domclick_detail_backfill + + +class _FakeRuns: + """Stub для scrape_runs: отслеживает вызовы, поддерживает cooperative cancel.""" + + def __init__(self, *, cancel: bool = False) -> None: + self.cancel = cancel + self.heartbeats: list[dict[str, int]] = [] + self.done: dict[str, int] | None = None + self.failed: tuple[str, dict[str, int]] | None = None + + def is_cancelled(self, _db: Any, _run_id: int) -> bool: + return self.cancel + + def update_heartbeat(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: + self.heartbeats.append(dict(counters)) + + def mark_done(self, _db: Any, _run_id: int, counters: dict[str, int]) -> None: + self.done = dict(counters) + + def mark_failed(self, _db: Any, _run_id: int, error: str, counters: dict[str, int]) -> None: + self.failed = (error, dict(counters)) + + +class _FakeBrowserFetcher: + """No-op async context manager standing in for BrowserFetcher (no browser launch).""" + + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + + async def __aenter__(self) -> _FakeBrowserFetcher: + return self + + async def __aexit__(self, *_args: Any) -> None: + return None + + +def _fake_db(rows: list[dict[str, Any]]) -> MagicMock: + """MagicMock Session whose execute(...).mappings().all() returns rows.""" + db = MagicMock() + db.execute.return_value.mappings.return_value.all.return_value = rows + return db + + +def _row(listing_id: int, offer_id: str) -> dict[str, Any]: + return { + "id": listing_id, + "source_id": offer_id, + "source_url": f"https://ekaterinburg.domclick.ru/card/sale__flat__{offer_id}", + } + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch: pytest.MonkeyPatch) -> None: + async def _instant(_secs: float) -> None: + return None + + monkeypatch.setattr(mod.asyncio, "sleep", _instant) + + +@pytest.fixture(autouse=True) +def _stub_browser(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(mod, "BrowserFetcher", _FakeBrowserFetcher) + + +def _install_runs(monkeypatch: pytest.MonkeyPatch, fake: _FakeRuns) -> None: + monkeypatch.setattr(mod.runs_mod, "is_cancelled", fake.is_cancelled) + monkeypatch.setattr(mod.runs_mod, "update_heartbeat", fake.update_heartbeat) + monkeypatch.setattr(mod.runs_mod, "mark_done", fake.mark_done) + monkeypatch.setattr(mod.runs_mod, "mark_failed", fake.mark_failed) + + +async def test_happy_path_enriches_all(monkeypatch: pytest.MonkeyPatch) -> None: + """Both rows enriched → mark_done(attempted=2, enriched=2), save called per row.""" + db = _fake_db([_row(1, "100"), _row(2, "101")]) + + fetch_mock = AsyncMock(return_value=MagicMock()) + save_mock = MagicMock(return_value=True) + monkeypatch.setattr(mod, "fetch_detail", fetch_mock) + monkeypatch.setattr(mod, "save_detail_enrichment", save_mock) + + fake = _FakeRuns() + _install_runs(monkeypatch, fake) + + counters = await run_domclick_detail_backfill(db, run_id=1, params={"request_delay_sec": 0}) + + assert counters.attempted == 2 + assert counters.enriched == 2 + assert counters.blocked == 0 + assert counters.failed == 0 + assert fetch_mock.await_count == 2 + assert save_mock.call_count == 2 + assert fake.done is not None + assert fake.done["attempted"] == 2 + assert fake.done["enriched"] == 2 + assert fake.failed is None + + +async def test_block_abort_marks_done_not_failed(monkeypatch: pytest.MonkeyPatch) -> None: + """DomClickBlockedError max_consecutive_blocks in a row → abort, mark_done (not failed).""" + db = _fake_db([_row(i, str(i)) for i in range(1, 11)]) # 10 rows available + + fetch_mock = AsyncMock(side_effect=DomClickBlockedError("datadome challenge")) + save_mock = MagicMock(return_value=True) + monkeypatch.setattr(mod, "fetch_detail", fetch_mock) + monkeypatch.setattr(mod, "save_detail_enrichment", save_mock) + + fake = _FakeRuns() + _install_runs(monkeypatch, fake) + + counters = await run_domclick_detail_backfill( + db, run_id=2, params={"request_delay_sec": 0, "max_consecutive_blocks": 5} + ) + + # Abort after 5 consecutive blocks → attempted=5, all blocked, none enriched. + assert counters.blocked == 5 + assert counters.attempted == 5 + assert counters.enriched == 0 + assert save_mock.call_count == 0 + # Block is transient → mark_done, NOT mark_failed. + assert fake.done is not None + assert fake.done["blocked"] == 5 + assert fake.failed is None + + +async def test_parse_failure_rolls_back_and_continues(monkeypatch: pytest.MonkeyPatch) -> None: + """DomClickParseError on row 1 → failed++, db.rollback; row 2 still enriched; mark_done.""" + db = _fake_db([_row(1, "100"), _row(2, "101")]) + + # First row -> parse error; second -> ok. + fetch_mock = AsyncMock(side_effect=[DomClickParseError("ssr schema drift"), MagicMock()]) + save_mock = MagicMock(return_value=True) + monkeypatch.setattr(mod, "fetch_detail", fetch_mock) + monkeypatch.setattr(mod, "save_detail_enrichment", save_mock) + + fake = _FakeRuns() + _install_runs(monkeypatch, fake) + + counters = await run_domclick_detail_backfill(db, run_id=3, params={"request_delay_sec": 0}) + + assert counters.attempted == 2 + assert counters.failed == 1 + assert counters.enriched == 1 + db.rollback.assert_called() # parse-failure rolled back the un-committed row + assert save_mock.call_count == 1 # only the second (parsed) row persisted + assert fake.done is not None + assert fake.failed is None + + +async def test_cancel_before_loop_returns_early(monkeypatch: pytest.MonkeyPatch) -> None: + """Cooperative cancel before loop → no snapshot SELECT, no fetch, no mark_done.""" + db = _fake_db([_row(1, "100")]) + + fetch_mock = AsyncMock(return_value=MagicMock()) + save_mock = MagicMock(return_value=True) + monkeypatch.setattr(mod, "fetch_detail", fetch_mock) + monkeypatch.setattr(mod, "save_detail_enrichment", save_mock) + + fake = _FakeRuns(cancel=True) + _install_runs(monkeypatch, fake) + + counters = await run_domclick_detail_backfill(db, run_id=4, params={"request_delay_sec": 0}) + + assert counters.attempted == 0 + assert fetch_mock.await_count == 0 + db.execute.assert_not_called() # cancel short-circuits before the snapshot SELECT + assert fake.done is None + assert fake.failed is None -- 2.45.3 From 480ecc90e525155e72419d5cdb12fb2343a6b9da Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 27 Jun 2026 18:24:08 +0300 Subject: [PATCH 2/8] =?UTF-8?q?feat(tradein/domclick):=20local=20Layer-B?= =?UTF-8?q?=20runner=20=E2=80=94=20home-IP=20+=20new-headless=20chromium,?= =?UTF-8?q?=20paced?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Self-contained script (playwright chromium, NO app imports) для сбора DomClick EKB detail с домашнего residential IP (обходит QRATOR — verified live: 2 карточки ok, repair/areas/priceHistory/egrn/owners собраны). Enumerate via bff (Layer A) + card __SSR_STATE__ parse (Layer B) → JSONL (resume/idempotent). KEY: дифференциатор — РЕЖИМ HEADLESS, не только IP. С того же IP старый headless=True → 403, new-headless → 200 + SSR. Раннер дефолтит на new-headless. Пейсинг: рандом 12-30с между карточками, render-jitter 6-10с, human-break каждые 25-40 (60-180с), 1 retry на 403. ~100-150 карточек/час, full ~6400 ≈ 45-65ч. JSONL ингестится в tradein DB отдельным шагом. НЕ deploy — локальный ops-инструмент. --- tradein-mvp/backend/scripts/README.md | 45 + .../backend/scripts/domclick_local_runner.py | 1194 +++++++++++++++++ 2 files changed, 1239 insertions(+) create mode 100644 tradein-mvp/backend/scripts/domclick_local_runner.py diff --git a/tradein-mvp/backend/scripts/README.md b/tradein-mvp/backend/scripts/README.md index 54d88559..854a8ed3 100644 --- a/tradein-mvp/backend/scripts/README.md +++ b/tradein-mvp/backend/scripts/README.md @@ -271,3 +271,48 @@ Caveats (also printed): CURRENT listings vs PAST deals (not point-in-time — needs `listing_source_snapshots` #570); asking-median + IQR core only; ДКП = registered price. Pure metric/ratio helpers are unit-tested in `tests/test_backtest_estimator.py` (no DB). + +--- + +## `domclick_local_runner.py` — DomClick EKB вторичка с домашнего IP (offline) + +**SELF-CONTAINED** (без `app.*` импортов, stdlib + `playwright.async_api`). +Запускается ОПЕРАТОРОМ на его машине с домашнего/резидентного IP — DomClick +фронтит QRATOR, который банит datacenter/mobile-proxy, но пропускает домашний. +Пишет JSONL (НЕ в БД); заливка в trade-in БД — отдельным ingest-шагом. + +Собирает Layer A (BFF search: `/api/offers/v1` + `/api/offers/count/v1`, +room×price-бакетинг под cap offset<2000) + Layer B (detail-карточка, +`window.__SSR_STATE__`: renovation, living/kitchen, priceHistory, egrnData +owners/collateral, sale_type, views/calls, wall/floor type, AVM). Логика +скопирована инлайн из `app/services/scrapers/domclick.py` + `domclick_detail.py`. + +### Режим окна (verified live 2026-06-27 на домашнем IP) + +QRATOR ловит СТАРЫЙ headless (`launch(headless=True)`) → `403 | Домклик` даже +с домашнего IP. NEW-headless (`--headless=new`) И headed → `200` + полный +`__SSR_STATE__`. Поэтому дефолт = new-headless (окно скрыто, детект проходит); +`--headed` = видимое окно (debug). BFF-enumerate проходит в любом режиме. + +### Темп (явное требование — НЕ спалить домашний IP) + +- пауза между карточками `--min-delay` 12с / `--max-delay` 30с (`random.uniform`); +- render-wait после goto `--render-min` 6с / `--render-max` 10с; +- «человеческий перерыв» каждые 25-40 карточек → 60-180с; +- на 403 — один ретрай через 45-90с, потом `status="blocked"` (не долбит). + +Дефолтный темп ≈ 100-150 карточек/час → полный свод ЕКБ (~6400) ≈ 45-65 ч +(несколько ночей). Скрипт РЕЗЮМИРУЕМ: при старте читает JSONL и пропускает +id с терминальным статусом (ok/blocked/parse_fail). + +```bash +cd tradein-mvp/backend +# боевой прогон (дефолтный медленный темп): +python scripts/domclick_local_runner.py --out domclick_ekb.jsonl +# быстрый smoke (маленькие задержки ТОЛЬКО для теста): +python scripts/domclick_local_runner.py --limit 2 --min-delay 3 --max-delay 5 --out dc_test.jsonl +# только перечислить (без detail): +python scripts/domclick_local_runner.py --enumerate-only --out enum.jsonl +``` + +Запускать python'ом, где установлен playwright (`python -c "import playwright"`). diff --git a/tradein-mvp/backend/scripts/domclick_local_runner.py b/tradein-mvp/backend/scripts/domclick_local_runner.py new file mode 100644 index 00000000..12213417 --- /dev/null +++ b/tradein-mvp/backend/scripts/domclick_local_runner.py @@ -0,0 +1,1194 @@ +#!/usr/bin/env python3 +"""DomClick EKB вторичка — SELF-CONTAINED локальный сборщик (search + detail). + +ЗАЧЕМ ОТДЕЛЬНЫЙ СКРИПТ +===================== +DomClick фронтит QRATOR anti-bot. Проверено вживую (2026-06-27): + • запрос с ДОМАШНЕГО (резидентного) IP → отдаёт полную карточку с + ``window.__SSR_STATE__`` (renovation / priceHistory / egrnData / AVM); + • запрос с datacenter / mobile-proxy IP → 403 | Домклик (bot_mitigation). +Поэтому продовый pipeline (BrowserFetcher через shared proxy) карточки НЕ +вытягивает. Этот скрипт запускается ОПЕРАТОРОМ на его собственной машине с +домашнего IP — без прокси — и собирает данные в JSONL, который потом +заливается в trade-in БД ОТДЕЛЬНЫМ шагом (этот скрипт в БД НЕ пишет). + +САМОДОСТАТОЧНОСТЬ +================= +Никаких ``app.*`` импортов: вся логика Layer A (BFF enumerate) и Layer B +(detail ``__SSR_STATE__`` parse) скопирована ИНЛАЙН из +``app/services/scrapers/domclick.py`` + ``domclick_detail.py`` + +``repair_state_normalizer.py``. Зависимости: stdlib + ``playwright.async_api``. + +ТРАНСПОРТ +========= +Playwright Chromium, БЕЗ прокси (домашний IP): + • enumerate (BFF JSON) — ``context.request.get(url)`` (тот же сетевой + стек/IP/куки браузера) → парсим первый ``{`` … последний ``}``; + • detail — ``page.goto(card_url)`` → jittered wait → ``page.evaluate`` с + ``fetch(location.href)`` чтобы получить СЫРОЙ HTML (SPA удаляет + ``window.__SSR_STATE__`` после гидрации, поэтому нужен повторный fetch). + +ВАЖНО — РЕЖИМ ОКНА (verified 2026-06-27 на домашнем IP этой машины) +------------------------------------------------------------------ +QRATOR ДЕТЕКТИТ СТАРЫЙ HEADLESS-РЕЖИМ: ``launch(headless=True)`` → даже с +домашнего IP карточка отдаёт ``401 / 403 | Домклик`` (нет ``__SSR_STATE__``). +А вот НОВЫЙ headless (``--headless=new``) И обычный headed → ``200`` + полный +``__SSR_STATE__`` (~330 КБ). Поэтому по умолчанию запускаемся в new-headless +(окно НЕ видно, но детект проходит). ``--headed`` → видимое окно (debug). +BFF-enumerate проходит в ЛЮБОМ режиме (это не браузерный рендер). + +ВАЖНО — ЧЕЛОВЕКОПОДОБНЫЙ ТЕМП (явное требование оператора) +========================================================= +Домашний IP — расходник: спалим частотой → бан и потеря единственного +рабочего канала. Поэтому темп НАМЕРЕННО медленный и настраиваемый: + • случайная пауза МЕЖДУ карточками: ``--min-delay`` (12с) / ``--max-delay`` + (30с), ``random.uniform``; + • jittered render-wait после goto: ``--render-min`` (6с) / ``--render-max`` + (10с); + • «человеческий перерыв»: каждые 25-40 карточек → сон 60-180с; + • на 403 — ОДИН ретрай через 45-90с; всё ещё блок → status="blocked" и + идём дальше (НЕ долбим). +При дефолтных задержках темп ≈ 100-150 карточек/час. Полный свод ЕКБ-вторички +(~6400 карточек) → ≈ 45-65 часов реального времени, т.е. несколько ночей. +Скрипт РЕЗЮМИРУЕМ: при старте читает уже собранный JSONL и пропускает готовые +id (status ok/blocked/parse_fail), так что прогон можно дробить на сессии. + +ЗАПУСК +====== +Нужен python с установленным playwright (``python -c "import playwright"``). +:: + + cd .../tradein-mvp/backend + # боевой прогон (дефолтный медленный темп), дописывает в JSONL: + python scripts/domclick_local_runner.py --out domclick_ekb.jsonl + # быстрый smoke (только для теста — маленькие задержки!): + python scripts/domclick_local_runner.py --limit 2 --min-delay 3 --max-delay 5 \ + --out /tmp/dc_test.jsonl + # только перечислить карточки, без detail: + python scripts/domclick_local_runner.py --enumerate-only --out enum.jsonl + +OUTPUT +====== +JSONL, по одной записи на карточку (append). Поля см. ``build_record``. +Файл потребляется ОТДЕЛЬНЫМ ingest-шагом в trade-in БД (listings + +offer_price_history); этот скрипт сам в БД ничего не пишет. +""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import logging +import random +import re +from datetime import UTC, datetime +from pathlib import Path +from typing import Any +from urllib.parse import urlencode + +from playwright.async_api import async_playwright + +logger = logging.getLogger("domclick_local") + +# ── API / гео константы (ground truth, verified 2026-06-27) ────────────────── + +_BFF_BASE = "https://bff-search-web.domclick.ru" +_EKB_ADDRESS_GUID = "0d475b79-88de-4054-818c-37d8f9d0d440" +_EKB_AREA_ID = "20561" +_CARD_HOST_FALLBACK = "https://ekaterinburg.domclick.ru" +_WARMUP_URL = "https://ekaterinburg.domclick.ru/" + +# Порядок бакетов комнат влияет на логи. urlencode("5+") → "5%2B". +ROOM_BUCKETS: tuple[str, ...] = ("st", "1", "2", "3", "4", "5+") + +OFFSET_CAP: int = 2000 # макс. offset, принимаемый BFF +PAGE_SIZE: int = 20 # items на страницу (жёстко задан API) +LTE_MAX: int = 1_000_000_000 # потолок цены для binary-split unbounded бакета +MIN_PRICE_SPAN: int = 100_000 # ниже этого span деление прекращаем +MAX_SPLIT_DEPTH: int = 30 # guard рекурсии binary-split (log2(1e9/1e5) ≈ 13) + +# Гео-guard ЕКБ (bbox). aids=20561 уже даёт чистый ЕКБ; bbox — defensive. +# Deviation от app: ослаблено строгое равенство offerRegionName до bbox-only +# (устойчивее к дрейфу имени региона; aids уже ограничивает выборку ЕКБ). +_EKB_LAT_MIN, _EKB_LAT_MAX = 56.6, 57.0 +_EKB_LON_MIN, _EKB_LON_MAX = 60.2, 60.9 + +# Маркеры QRATOR / anti-bot challenge. +_BLOCK_MARKERS: tuple[str, ...] = ( + "qrator", + "bot_mitigation", + "система защиты", + "403 | домклик", + "captcha", + "datadome", + "access denied", +) + +# Реалистичный desktop-Chrome UA (headless-UA палится anti-bot'ами). +_USER_AGENT = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" +) + +# SSR-стейт встроен как JS-литерал (НЕ чистый JSON: bare `undefined`). +_SSR_ASSIGN_RE = re.compile(r"window\.__SSR_STATE__\s*=\s*") +_UNDEFINED_RE = re.compile(r"(?<=[:\[,\s])undefined(?=[\s,\]\}])") +_ITEM_ID_RE = re.compile(r"sale__flat__(\d+)") + +# JS для повторного fetch сырого HTML карточки из контекста страницы. +_FETCH_RAW_JS = ( + "async () => { const r = await fetch(window.location.href, " + "{credentials: 'include'}); return await r.text(); }" +) + + +# ── Исключения ─────────────────────────────────────────────────────────────── + + +class DomClickBlockedError(Exception): + """Anti-bot challenge (QRATOR/DataDome/403) — НЕ parse-failure.""" + + +class DomClickParseError(Exception): + """HTTP 200, но __SSR_STATE__ не найден / не парсится (дрейф схемы).""" + + +class _EnumLimitReachedError(Exception): + """Внутренний сигнал: набрали target новых карточек — стоп enumerate.""" + + +# ── repair_state normalizer (инлайн из repair_state_normalizer.py) ─────────── + +_RAW_TO_ENUM: dict[str, str] = { + "required": "needs_repair", + "without": "needs_repair", + "no": "needs_repair", + "rough": "needs_repair", + "prefine": "needs_repair", + "cosmetic": "standard", + "euro": "good", + "fine": "good", + "designer": "excellent", + "design": "excellent", +} +_VALID_ENUM: frozenset[str] = frozenset({"needs_repair", "standard", "good", "excellent"}) + +# renovation (RU) → каноничный enum (локальная карта Layer B). +# Расширено относительно app-версии РЕАЛЬНЫМИ короткими токенами DomClick, +# наблюдёнными вживую 2026-06-27 ("Евро", "Косметический"): SSR-поле renovation +# отдаёт КОРОТКУЮ форму ("Евро"), а не "евроремонт" — иначе good терялся в None. +_RENOVATION_MAP: dict[str, str] = { + "евроремонт": "good", + "евро": "good", + "косметический": "standard", + "косметический ремонт": "standard", + "дизайнерский": "excellent", + "дизайнерский ремонт": "excellent", + "черновая": "needs_repair", + "черновая отделка": "needs_repair", + "без отделки": "needs_repair", + "без ремонта": "needs_repair", + "требует ремонта": "needs_repair", + "предчистовая": "needs_repair", + "предчистовая отделка": "needs_repair", +} + +_TEXT_PATTERNS: tuple[tuple[re.Pattern[str], str], ...] = ( + ( + re.compile( + r"дизайнерск(?:ий|ого|ом|ая)\s+ремонт" + r"|дизайнерск(?:ая|ой)\s+отделк" + r"|премиальн\w*\s+ремонт" + r"|эксклюзивн\w*\s+ремонт" + r"|авторск\w*\s+ремонт", + re.IGNORECASE, + ), + "excellent", + ), + ( + re.compile( + r"евроремонт" + r"|евро[\s-]?ремонт" + r"|отличн\w*\s+ремонт" + r"|свеж\w*\s+ремонт" + r"|современн\w*\s+ремонт" + r"|качественн\w*\s+ремонт" + r"|ремонт\s+в\s+отличн", + re.IGNORECASE, + ), + "good", + ), + ( + re.compile( + r"без\s+отделк" + r"|без\s+ремонт" + r"|черновая\s+отделк" + r"|черновой\s+ремонт" + r"|требует(?:ся)?\s+ремонт" + r"|требует(?:ся)?\s+космет" + r"|нужен\s+ремонт" + r"|под\s+ремонт" + r"|под\s+чистов\w+\s+отделк" + r"|предчистов", + re.IGNORECASE, + ), + "needs_repair", + ), + ( + re.compile( + r"косметическ\w*\s+ремонт" + r"|косметическ\w*\s+отделк" + r"|обычн\w*\s+ремонт" + r"|жил(?:ое|ом)\s+состоян" + r"|хорош\w*\s+состоян", + re.IGNORECASE, + ), + "standard", + ), +) + + +def normalize_repair_state(raw: str | None) -> str | None: + """raw repair → каноничный enum (идемпотентно). Неизвестное → None.""" + if raw is None: + return None + if raw in _VALID_ENUM: + return raw + result = _RAW_TO_ENUM.get(raw) or _RAW_TO_ENUM.get(raw.lower()) + if result is None: + logger.debug("repair_state: unknown raw %r — stored NULL", raw) + return result + + +def infer_repair_state_from_text(description: str | None) -> str | None: + """repair_state из текста описания (fallback). Первое совпадение по силе.""" + if not description: + return None + for pattern, enum_value in _TEXT_PATTERNS: + if pattern.search(description): + return enum_value + return None + + +def _map_repair_state(renovation: str | None) -> str | None: + """RU renovation → enum; нет совпадения в карте → инференс из текста.""" + if not renovation: + return None + mapped = _RENOVATION_MAP.get(renovation.strip().lower()) + if mapped is not None: + return normalize_repair_state(mapped) + return infer_repair_state_from_text(renovation) + + +# ── value coercion (инлайн из domclick_detail.py) ──────────────────────────── + + +def _to_float(value: Any) -> float | None: + if value is None or isinstance(value, bool): + return None + try: + return float(value) + except (ValueError, TypeError): + return None + + +def _to_int(value: Any) -> int | None: + if value is None or isinstance(value, bool): + return None + try: + return int(value) + except (ValueError, TypeError): + return None + + +def _pos_float(value: Any) -> float | None: + """float > 0, иначе None. DomClick эмитит 0 как sentinel «неизвестно» + для living/kitchen area (особенно у студий) — 0.0 ввело бы в заблуждение.""" + f = _to_float(value) + if f is None or f <= 0: + return None + return f + + +def _parse_change_time(value: Any) -> datetime | None: + """date изменения цены → tz-aware datetime. epoch(сек/мс)/ISO8601/num-str.""" + if value is None or isinstance(value, bool): + return None + if isinstance(value, int | float): + ts = float(value) + if ts > 1e12: + ts /= 1000.0 + try: + return datetime.fromtimestamp(ts, tz=UTC) + except (ValueError, OSError, OverflowError): + return None + s = str(value).strip() + if not s: + return None + if s.isdigit(): + ts = float(s) + if ts > 1e12: + ts /= 1000.0 + try: + return datetime.fromtimestamp(ts, tz=UTC) + except (ValueError, OSError, OverflowError): + return None + try: + dt = datetime.fromisoformat(s.replace("Z", "+00:00")) + except (ValueError, TypeError): + return None + if dt.tzinfo is None: + dt = dt.replace(tzinfo=UTC) + return dt + + +def _compact(d: dict[str, Any]) -> dict[str, Any]: + return {k: v for k, v in d.items() if v is not None} + + +def _pick(d: dict[str, Any], *keys: str) -> Any: + for k in keys: + v = d.get(k) + if v is not None: + return v + return None + + +def _extract_item_id(url: str) -> str | None: + m = _ITEM_ID_RE.search(url or "") + return m.group(1) if m else None + + +# ── BFF JSON extraction (инлайн из domclick.py) ────────────────────────────── + + +def _is_block_text(text: str) -> bool: + low = text.lower() + return any(m in low for m in _BLOCK_MARKERS) + + +def _extract_json(body: str) -> dict[str, Any]: + """Первый ``{`` … последний ``}`` из ответа BFF (raw JSON или 
-wrap).
+
+    Raises:
+        DomClickBlockedError: QRATOR/captcha маркеры в теле.
+        ValueError: JSON не найден / не dict.
+    """
+    if _is_block_text(body):
+        raise DomClickBlockedError("BFF: QRATOR block page detected")
+    start = body.find("{")
+    end = body.rfind("}")
+    if start == -1 or end == -1 or end <= start:
+        raise ValueError(f"No JSON object in BFF response (len={len(body)})")
+    data = json.loads(body[start : end + 1])
+    if not isinstance(data, dict):
+        raise ValueError(f"BFF JSON is not a dict: {type(data)}")
+    return data
+
+
+def _build_offers_url(rooms: str, gte: int | None, lte: int | None, offset: int) -> str:
+    params: list[tuple[str, str]] = [
+        ("address", _EKB_ADDRESS_GUID),
+        ("aids", _EKB_AREA_ID),
+        ("deal_type", "sale"),
+        ("category", "living"),
+        ("offer_type", "flat"),
+        ("rooms", rooms),
+        ("sort", "qi"),
+        ("sort_dir", "desc"),
+        ("offset", str(offset)),
+        ("limit", str(PAGE_SIZE)),
+    ]
+    if gte is not None:
+        params.append(("sale_price__gte", str(gte)))
+    if lte is not None:
+        params.append(("sale_price__lte", str(lte)))
+    return f"{_BFF_BASE}/api/offers/v1?{urlencode(params)}"
+
+
+def _build_count_url(rooms: str, gte: int | None, lte: int | None) -> str:
+    params: list[tuple[str, str]] = [
+        ("address", _EKB_ADDRESS_GUID),
+        ("aids", _EKB_AREA_ID),
+        ("deal_type", "sale"),
+        ("category", "living"),
+        ("offer_type", "flat"),
+        ("rooms", rooms),
+        ("sort", "qi"),
+        ("sort_dir", "desc"),
+    ]
+    if gte is not None:
+        params.append(("sale_price__gte", str(gte)))
+    if lte is not None:
+        params.append(("sale_price__lte", str(lte)))
+    return f"{_BFF_BASE}/api/offers/count/v1?{urlencode(params)}"
+
+
+def _is_geo_ok(item: dict[str, Any]) -> bool:
+    """bbox-guard ЕКБ. lat/lon обязательны и в пределах bbox."""
+    loc = item.get("location") or {}
+    lat = loc.get("lat")
+    lon = loc.get("lon")
+    if lat is None or lon is None:
+        return False
+    try:
+        latf, lonf = float(lat), float(lon)
+    except (ValueError, TypeError):
+        return False
+    return _EKB_LAT_MIN <= latf <= _EKB_LAT_MAX and _EKB_LON_MIN <= lonf <= _EKB_LON_MAX
+
+
+# ── SERP item → плоский dict (инлайн из domclick.py _map_item) ────────────────
+
+
+def map_serp_item(item: dict[str, Any], force_rooms: int | None) -> dict[str, Any] | None:
+    """offer-item из BFF → SERP-dict. None при price<=0 / без id / ошибке."""
+    try:
+        price = item.get("price") or 0
+        if price <= 0:
+            return None
+        item_id = item.get("id")
+        if item_id is None:
+            return None
+        sid = str(item_id)
+
+        path = item.get("path") or ""
+        if path.startswith("/"):
+            path = _CARD_HOST_FALLBACK + path
+
+        loc = item.get("location") or {}
+        oi = item.get("objectInfo") or {}
+        house = item.get("house") or {}
+        addr = item.get("address") or {}
+
+        area_raw = oi.get("area")
+        floor_raw = oi.get("floor")
+        floors_raw = house.get("floors")
+        year_raw = house.get("buildYear")
+        sq_raw = item.get("squarePrice")
+
+        if force_rooms is not None:
+            rooms: int | None = force_rooms
+        else:
+            rr = oi.get("rooms")
+            rooms = int(rr) if rr is not None else None
+
+        return {
+            "id": sid,
+            "source_url": path,
+            "price_rub": int(price),
+            "square_price": int(sq_raw) if sq_raw else None,
+            "area_m2": float(area_raw) if area_raw else None,
+            "rooms": rooms,
+            "floor": int(floor_raw) if floor_raw else None,
+            "total_floors": int(floors_raw) if floors_raw else None,
+            "year_built": int(year_raw) if year_raw else None,
+            "lat": loc.get("lat"),
+            "lon": loc.get("lon"),
+            "address": addr.get("displayName"),
+            "is_apartment": oi.get("isApartment"),
+            "offer_region_name": item.get("offerRegionName"),
+            "description": item.get("description"),
+            "is_sber_collateral": item.get("isSberCollateral"),
+            "published_date": item.get("publishedDate"),
+        }
+    except Exception:
+        logger.warning("map_serp_item failed id=%r", item.get("id"), exc_info=True)
+        return None
+
+
+# ── SSR state extraction (инлайн из domclick_detail.py) ──────────────────────
+
+
+def _scan_balanced_object(s: str, start: int) -> str:
+    """От '{' по индексу start до парной '}', игнорируя скобки внутри строк."""
+    depth = 0
+    in_string = False
+    quote_char = ""
+    escaped = False
+    for i in range(start, len(s)):
+        ch = s[i]
+        if in_string:
+            if escaped:
+                escaped = False
+            elif ch == "\\":
+                escaped = True
+            elif ch == quote_char:
+                in_string = False
+            continue
+        if ch in ('"', "'"):
+            in_string = True
+            quote_char = ch
+        elif ch == "{":
+            depth += 1
+        elif ch == "}":
+            depth -= 1
+            if depth == 0:
+                return s[start : i + 1]
+    raise DomClickParseError("__SSR_STATE__ object not balanced (no matching brace)")
+
+
+def _extract_ssr_state(html: str) -> dict[str, Any]:
+    """window.__SSR_STATE__ object-литерал из HTML карточки → dict.
+
+    Raises:
+        DomClickBlockedError: нет стейта + anti-bot маркеры.
+        DomClickParseError: стейт не найден / не сбалансирован / json.loads упал.
+    """
+    match = _SSR_ASSIGN_RE.search(html)
+    if match is None:
+        if _is_block_text(html):
+            raise DomClickBlockedError("detail: challenge page (no __SSR_STATE__)")
+        raise DomClickParseError("__SSR_STATE__ not found")
+    brace_start = html.find("{", match.end())
+    if brace_start == -1:
+        raise DomClickParseError("__SSR_STATE__ assignment has no opening '{'")
+    literal = _scan_balanced_object(html, brace_start)
+    sanitized = _UNDEFINED_RE.sub("null", literal)
+    try:
+        data = json.loads(sanitized)
+    except json.JSONDecodeError as exc:
+        raise DomClickParseError(
+            f"__SSR_STATE__ json.loads failed: {exc} (head={sanitized[:160]!r})"
+        ) from exc
+    if not isinstance(data, dict):
+        raise DomClickParseError(f"__SSR_STATE__ is not an object: {type(data)}")
+    return data
+
+
+def _extract_owners_count(egrn: dict[str, Any]) -> int | None:
+    """egrnData.owners_count. Ground truth: {status, value}; fallback на скаляр."""
+    oc = egrn.get("owners_count")
+    if oc is None:
+        oc = egrn.get("ownersCount")
+    if isinstance(oc, dict):
+        return _to_int(oc.get("value"))
+    return _to_int(oc)
+
+
+def parse_detail_html(html: str, source_url: str) -> dict[str, Any]:
+    """Парсер detail-карточки → dict обогащения. _extract_ssr_state может бросить
+    Blocked/ParseError (propagate); навигация по полям обёрнута → минимум при сюрпризе.
+    """
+    state = _extract_ssr_state(html)
+    try:
+        pc = state.get("productCard") or {}
+        oi = pc.get("objectInfo") or {}
+        hi = (state.get("houseInfo") or {}).get("info") or {}
+        egrn = pc.get("egrnData") or {}
+
+        renovation = oi.get("renovation")
+        repair_raw = renovation if isinstance(renovation, str) else None
+        repair_state = _map_repair_state(repair_raw)
+
+        living = _pos_float(oi.get("livingArea"))
+        kitchen = _pos_float(oi.get("kitchenArea"))
+
+        sale_type = (pc.get("legalOptions") or {}).get("saleType")
+        owners_count = _extract_owners_count(egrn)
+
+        collateral = egrn.get("collateral")
+        collateral_sber = egrn.get("collateral_sber")
+        if egrn:
+            encumbrances_clean: bool | None = not bool(collateral or collateral_sber)
+        else:
+            encumbrances_clean = None
+
+        wall_type = hi.get("wallType") or (pc.get("house") or {}).get("wallType")
+        floor_type = hi.get("floorType")
+        year_built = _to_int(hi.get("buildYear"))
+        views = _to_int(pc.get("viewsCount"))
+        calls = _to_int(pc.get("callsCount"))
+
+        # priceHistory → нормализованные записи (skip при битой дате/без цены).
+        raw_history = (pc.get("priceInfo") or {}).get("priceHistory") or []
+        price_changes: list[dict[str, Any]] = []
+        if isinstance(raw_history, list):
+            for entry in raw_history:
+                if not isinstance(entry, dict):
+                    continue
+                ct = _parse_change_time(entry.get("date"))
+                price = _to_int(entry.get("price"))
+                if ct is None or price is None:
+                    continue
+                diff = entry.get("diff")
+                diff_pct = (
+                    diff if isinstance(diff, int | float) and not isinstance(diff, bool) else None
+                )
+                price_changes.append(
+                    {
+                        "change_time": ct.isoformat(),
+                        "price_rub": price,
+                        "diff_percent": diff_pct,
+                        "state": entry.get("state"),
+                    }
+                )
+
+        # AVM — top-level pricePrediction (часто пусто); defensively разворачиваем.
+        avm_root = state.get("pricePrediction") or {}
+        avm_src: Any = avm_root
+        if isinstance(avm_root, dict):
+            avm_src = avm_root.get("result") or avm_root.get("data") or avm_root
+        if not isinstance(avm_src, dict):
+            avm_src = {}
+        avm = _compact(
+            {
+                "market_price": _pick(avm_src, "market_price", "marketPrice"),
+                "min": _pick(avm_src, "min_market_price", "minMarketPrice"),
+                "max": _pick(avm_src, "max_market_price", "maxMarketPrice"),
+                "repair_quality": _pick(avm_src, "repair_quality", "repairQuality"),
+            }
+        )
+
+        demand = _compact(
+            {
+                "calls": calls,
+                "favorites": pc.get("favoriteOfferUsersCount"),
+                "duplicates": pc.get("duplicatesOfferCount"),
+            }
+        )
+        raw_extra = _compact(
+            {
+                "wall_type": wall_type,
+                "floor_type": floor_type,
+                "entrance_count": hi.get("entranceCount"),
+                "quarters_count": hi.get("quartersCount"),
+                "energy_efficiency": hi.get("energyEfficiency"),
+                "building_series": hi.get("buildingSeries"),
+                "domclick_building_guid": (pc.get("address") or {}).get("guid"),
+                "demand": demand or None,
+                "collateral": collateral,
+                "collateral_sber": collateral_sber,
+            }
+        )
+
+        return {
+            "repair_state": repair_state,
+            "repair_raw": repair_raw,
+            "living_area_m2": living,
+            "kitchen_area_m2": kitchen,
+            "sale_type": sale_type if isinstance(sale_type, str) else None,
+            "owners_count": owners_count,
+            "encumbrances_clean": encumbrances_clean,
+            "wall_type": wall_type,
+            "floor_type": floor_type,
+            "views": views,
+            "calls": calls,
+            "price_changes": price_changes,
+            "collateral_sber": bool(collateral_sber) if collateral_sber is not None else None,
+            "year_built": year_built,
+            "avm": avm or None,
+            "raw_extra": raw_extra,
+        }
+    except (DomClickBlockedError, DomClickParseError):
+        raise
+    except Exception:
+        logger.warning("detail field navigation failed for %s — minimal", source_url, exc_info=True)
+        return {}
+
+
+# ── Output record ────────────────────────────────────────────────────────────
+
+
+def build_record(
+    serp: dict[str, Any], detail: dict[str, Any] | None, status: str
+) -> dict[str, Any]:
+    """SERP + detail → итоговая JSONL-запись."""
+    d = detail or {}
+    repair_state = d.get("repair_state")
+    if not repair_state:
+        # fallback-инференс из SERP-описания (как в проде для пустого структурного)
+        repair_state = infer_repair_state_from_text(serp.get("description"))
+
+    is_sber = serp.get("is_sber_collateral")
+    if not is_sber and d.get("collateral_sber"):
+        is_sber = True
+
+    return {
+        "id": serp.get("id"),
+        "source_url": serp.get("source_url"),
+        "status": status,
+        "fetched_at": datetime.now(UTC).isoformat(),
+        "price_rub": serp.get("price_rub"),
+        "area_m2": serp.get("area_m2"),
+        "rooms": serp.get("rooms"),
+        "floor": serp.get("floor"),
+        "total_floors": serp.get("total_floors"),
+        "year_built": d.get("year_built") or serp.get("year_built"),
+        "lat": serp.get("lat"),
+        "lon": serp.get("lon"),
+        "address": serp.get("address"),
+        "repair_state": repair_state,
+        "repair_raw": d.get("repair_raw"),
+        "living_area_m2": d.get("living_area_m2"),
+        "kitchen_area_m2": d.get("kitchen_area_m2"),
+        "sale_type": d.get("sale_type"),
+        "owners_count": d.get("owners_count"),
+        "encumbrances_clean": d.get("encumbrances_clean"),
+        "wall_type": d.get("wall_type"),
+        "floor_type": d.get("floor_type"),
+        "views": d.get("views"),
+        "calls": d.get("calls"),
+        "price_changes": d.get("price_changes") or [],
+        "is_sber_collateral": is_sber,
+        "avm": d.get("avm") or {},
+        "raw_extra": {
+            "square_price": serp.get("square_price"),
+            "is_apartment": serp.get("is_apartment"),
+            "offer_region_name": serp.get("offer_region_name"),
+            "published_date": serp.get("published_date"),
+            **(d.get("raw_extra") or {}),
+        },
+    }
+
+
+# ── Resume / output IO ───────────────────────────────────────────────────────
+
+_TERMINAL_STATUSES = frozenset({"ok", "blocked", "parse_fail"})
+
+
+def load_done_ids(path: Path) -> set[str]:
+    """id с терминальным detail-статусом из существующего JSONL (для resume)."""
+    done: set[str] = set()
+    if not path.exists():
+        return done
+    with path.open("r", encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            rid = rec.get("id")
+            if rid and rec.get("status") in _TERMINAL_STATUSES:
+                done.add(str(rid))
+    return done
+
+
+def append_record(path: Path, rec: dict[str, Any]) -> None:
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(json.dumps(rec, ensure_ascii=False, default=str) + "\n")
+        fh.flush()
+
+
+# ── BFF transport (playwright APIRequestContext) ─────────────────────────────
+
+
+async def fetch_bff_json(context: Any, url: str) -> dict[str, Any]:
+    """GET BFF через context.request (тот же IP/куки браузера) → dict."""
+    resp = await context.request.get(
+        url,
+        headers={
+            "Accept": "application/json, text/plain, */*",
+            "Referer": "https://ekaterinburg.domclick.ru/search?deal_type=sale",
+            "Accept-Language": "ru-RU,ru;q=0.9",
+        },
+        timeout=60000,
+    )
+    body = await resp.text()
+    if resp.status == 403:
+        raise DomClickBlockedError(f"BFF 403 for {url}")
+    return _extract_json(body)
+
+
+async def get_count(context: Any, rooms: str, gte: int | None, lte: int | None) -> int:
+    data = await fetch_bff_json(context, _build_count_url(rooms, gte, lte))
+    res = data.get("result") or {}
+    raw = res.get("snippetsCount")
+    return int(raw) if raw else 0
+
+
+# ── Enumerate (mirror domclick.py _sweep_bucket / _paginate) ─────────────────
+
+
+async def _paginate(
+    context: Any,
+    rooms: str,
+    gte: int | None,
+    lte: int | None,
+    seen: set[str],
+    done: set[str],
+    out: list[dict[str, Any]],
+    limit_pages: int,
+    target_new: int | None,
+    enum_jitter: tuple[float, float],
+) -> None:
+    force_rooms = 0 if rooms == "st" else None
+    page_idx = 0
+    while page_idx < limit_pages:
+        offset = page_idx * PAGE_SIZE
+        if offset >= OFFSET_CAP:
+            break
+        data = await fetch_bff_json(context, _build_offers_url(rooms, gte, lte, offset))
+        items = (data.get("result") or {}).get("items") or []
+        if not items:
+            break
+        for item in items:
+            serp = map_serp_item(item, force_rooms)
+            if serp is None:
+                continue
+            if not _is_geo_ok(item):
+                continue
+            sid = serp["id"]
+            if sid in seen or sid in done:
+                continue
+            seen.add(sid)
+            out.append(serp)
+            if target_new is not None and len(out) >= target_new:
+                raise _EnumLimitReachedError
+        page_idx += 1
+        if len(items) < PAGE_SIZE:
+            break
+        await asyncio.sleep(random.uniform(*enum_jitter))
+
+
+async def _sweep_bucket(
+    context: Any,
+    rooms: str,
+    gte: int | None,
+    lte: int | None,
+    seen: set[str],
+    done: set[str],
+    out: list[dict[str, Any]],
+    limit_pages: int,
+    target_new: int | None,
+    enum_jitter: tuple[float, float],
+    depth: int,
+) -> None:
+    count = await get_count(context, rooms, gte, lte)
+    if count == 0:
+        return
+    if count > OFFSET_CAP and depth < MAX_SPLIT_DEPTH:
+        _gte = gte if gte is not None else 0
+        _lte = lte if lte is not None else LTE_MAX
+        span = _lte - _gte
+        if span <= MIN_PRICE_SPAN:
+            logger.warning(
+                "rooms=%r price=[%s,%s] count=%d > cap but span<=%d — paginating as-is",
+                rooms,
+                gte,
+                lte,
+                count,
+                MIN_PRICE_SPAN,
+            )
+            await _paginate(
+                context, rooms, gte, lte, seen, done, out, limit_pages, target_new, enum_jitter
+            )
+            return
+        mid = (_gte + _lte) // 2
+        logger.info(
+            "rooms=%r count=%d > cap — split [%d,%d]+[%d,%d]",
+            rooms,
+            count,
+            _gte,
+            mid,
+            mid + 1,
+            _lte,
+        )
+        await _sweep_bucket(
+            context,
+            rooms,
+            _gte,
+            mid,
+            seen,
+            done,
+            out,
+            limit_pages,
+            target_new,
+            enum_jitter,
+            depth + 1,
+        )
+        await _sweep_bucket(
+            context,
+            rooms,
+            mid + 1,
+            _lte,
+            seen,
+            done,
+            out,
+            limit_pages,
+            target_new,
+            enum_jitter,
+            depth + 1,
+        )
+    else:
+        await _paginate(
+            context, rooms, gte, lte, seen, done, out, limit_pages, target_new, enum_jitter
+        )
+
+
+async def enumerate_cards(
+    context: Any,
+    done: set[str],
+    limit_pages: int,
+    target_new: int | None,
+    enum_jitter: tuple[float, float],
+) -> list[dict[str, Any]]:
+    """Свод ЕКБ-вторички через BFF: все room-бакеты, dedup, skip done."""
+    out: list[dict[str, Any]] = []
+    seen: set[str] = set()
+    try:
+        for bucket in ROOM_BUCKETS:
+            logger.info("enumerate: bucket rooms=%r (collected=%d)", bucket, len(out))
+            await _sweep_bucket(
+                context,
+                bucket,
+                None,
+                None,
+                seen,
+                done,
+                out,
+                limit_pages,
+                target_new,
+                enum_jitter,
+                0,
+            )
+    except _EnumLimitReachedError:
+        logger.info("enumerate: reached target=%d new cards — stop sweep", target_new)
+    except DomClickBlockedError as exc:
+        logger.error(
+            "enumerate: QRATOR block (%s) — aborting sweep, keeping %d collected", exc, len(out)
+        )
+    logger.info("enumerate: done, %d new cards collected", len(out))
+    return out
+
+
+# ── Detail transport ─────────────────────────────────────────────────────────
+
+
+async def fetch_card_html(
+    page: Any, url: str, render_wait: tuple[float, float]
+) -> tuple[str, str, int]:
+    """goto → jittered render-wait → re-fetch сырого HTML. (html, title, status)."""
+    resp = await page.goto(url, wait_until="domcontentloaded", timeout=70000)
+    await asyncio.sleep(random.uniform(*render_wait))
+    title = ""
+    try:
+        title = (await page.title()) or ""
+    except Exception:
+        title = ""
+    html = await page.evaluate(_FETCH_RAW_JS)
+    status = resp.status if resp else 0
+    return html, title, status
+
+
+async def process_card(
+    page: Any,
+    serp: dict[str, Any],
+    render_wait: tuple[float, float],
+) -> tuple[str, dict[str, Any] | None]:
+    """Один detail-fetch с одним ретраем на блок. → (status, detail|None)."""
+    url = serp["source_url"]
+    for attempt in (1, 2):
+        try:
+            html, title, http_status = await fetch_card_html(page, url, render_wait)
+            if http_status == 403 or _is_block_text(title) or _is_block_text(html[:4096]):
+                raise DomClickBlockedError(f"403/challenge title={title!r}")
+            detail = parse_detail_html(html, url)
+            return "ok", detail
+        except DomClickBlockedError as exc:
+            if attempt == 1:
+                wait = random.uniform(45, 90)
+                logger.warning(
+                    "card %s blocked (%s) — retry once after %.0fs", serp["id"], exc, wait
+                )
+                await asyncio.sleep(wait)
+                continue
+            logger.warning("card %s still blocked after retry — status=blocked", serp["id"])
+            return "blocked", None
+        except DomClickParseError as exc:
+            logger.warning("card %s parse_fail: %s", serp["id"], exc)
+            return "parse_fail", None
+        except Exception as exc:
+            if attempt == 1:
+                logger.warning("card %s fetch error (%s) — retry once", serp["id"], exc)
+                await asyncio.sleep(random.uniform(8, 15))
+                continue
+            logger.warning("card %s fetch error after retry — parse_fail", serp["id"])
+            return "parse_fail", None
+    return "parse_fail", None
+
+
+# ── Orchestration ────────────────────────────────────────────────────────────
+
+
+class Summary:
+    """Счётчики прогона (для финального лога + KeyboardInterrupt flush)."""
+
+    def __init__(self) -> None:
+        self.enumerated = 0
+        self.ok = 0
+        self.blocked = 0
+        self.parse_fail = 0
+        self.skipped_done = 0
+        self.started = datetime.now(UTC)
+
+    def log_final(self) -> None:
+        elapsed = (datetime.now(UTC) - self.started).total_seconds()
+        logger.info(
+            "SUMMARY enumerated=%d ok=%d blocked=%d parse_fail=%d skipped_done=%d elapsed=%.0fs",
+            self.enumerated,
+            self.ok,
+            self.blocked,
+            self.parse_fail,
+            self.skipped_done,
+            elapsed,
+        )
+
+
+async def run(args: argparse.Namespace) -> None:
+    out_path = Path(args.out)
+    out_path.parent.mkdir(parents=True, exist_ok=True)
+
+    done = load_done_ids(out_path)
+    logger.info("resume: %d ids already done in %s", len(done), out_path)
+
+    summary = Summary()
+    render_wait = (args.render_min, args.render_max)
+    target_new = args.limit if args.limit and args.limit > 0 else None
+    enum_jitter = (1.5, 4.0)
+
+    # ВАЖНО: launch(headless=True) = СТАРЫЙ headless → QRATOR 403 даже с дом. IP.
+    # Запускаемся с headless=False, но добавляем --headless=new (новый headless,
+    # окно не видно, детект проходит). --headed → без флага = видимое окно.
+    launch_args = ["--disable-blink-features=AutomationControlled"]
+    if not args.headed:
+        launch_args.append("--headless=new")
+
+    async with async_playwright() as pw:
+        browser = await pw.chromium.launch(headless=False, args=launch_args)
+        context = await browser.new_context(
+            user_agent=_USER_AGENT,
+            locale="ru-RU",
+            timezone_id="Asia/Yekaterinburg",
+            viewport={"width": 1366, "height": 768},
+            extra_http_headers={"Accept-Language": "ru-RU,ru;q=0.9"},
+        )
+        page = await context.new_page()
+
+        # Warm-up: устанавливаем QRATOR-куки .domclick.ru перед BFF-запросами.
+        try:
+            await page.goto(_WARMUP_URL, wait_until="domcontentloaded", timeout=70000)
+            await asyncio.sleep(random.uniform(2.0, 4.0))
+            logger.info("warmup: loaded %s", _WARMUP_URL)
+        except Exception as exc:
+            logger.warning("warmup navigation failed (%s) — continuing", exc)
+
+        # 1. Enumerate.
+        cards = await enumerate_cards(context, done, args.limit_pages, target_new, enum_jitter)
+        summary.enumerated = len(cards)
+
+        if args.enumerate_only:
+            logger.info("enumerate-only: writing %d SERP records (status=enumerated)", len(cards))
+            for serp in cards:
+                append_record(out_path, build_record(serp, None, "enumerated"))
+            summary.log_final()
+            await context.close()
+            await browser.close()
+            return
+
+        # 2. Detail.
+        next_break_at = random.randint(25, 40)
+        processed_since_break = 0
+        try:
+            for idx, serp in enumerate(cards, start=1):
+                if args.limit and (summary.ok + summary.blocked + summary.parse_fail) >= args.limit:
+                    break
+                sid = serp["id"]
+                if sid in done:
+                    summary.skipped_done += 1
+                    continue
+
+                status, detail = await process_card(page, serp, render_wait)
+                rec = build_record(serp, detail, status)
+                append_record(out_path, rec)
+                done.add(sid)
+
+                if status == "ok":
+                    summary.ok += 1
+                elif status == "blocked":
+                    summary.blocked += 1
+                else:
+                    summary.parse_fail += 1
+
+                logger.info(
+                    "[%d/%d] id=%s status=%s repair=%s living=%s owners=%s price_changes=%d "
+                    "| totals ok=%d blocked=%d fail=%d",
+                    idx,
+                    len(cards),
+                    sid,
+                    status,
+                    rec.get("repair_state"),
+                    rec.get("living_area_m2"),
+                    rec.get("owners_count"),
+                    len(rec.get("price_changes") or []),
+                    summary.ok,
+                    summary.blocked,
+                    summary.parse_fail,
+                )
+
+                processed_since_break += 1
+                if processed_since_break >= next_break_at:
+                    brk = random.uniform(60, 180)
+                    logger.info(
+                        "human break: sleeping %.0fs after %d cards", brk, processed_since_break
+                    )
+                    await asyncio.sleep(brk)
+                    processed_since_break = 0
+                    next_break_at = random.randint(25, 40)
+
+                # Пауза между карточками (главный темп-контроль).
+                await asyncio.sleep(random.uniform(args.min_delay, args.max_delay))
+        except (KeyboardInterrupt, asyncio.CancelledError):
+            logger.warning("interrupted — flushing summary and exiting gracefully")
+        finally:
+            summary.log_final()
+            await context.close()
+            await browser.close()
+
+
+def build_arg_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        description="DomClick EKB вторичка local runner (home IP, human-paced).",
+    )
+    p.add_argument("--out", default="domclick_ekb.jsonl", help="JSONL output (append, resumable)")
+    p.add_argument("--limit", type=int, default=None, help="cap total cards (для теста)")
+    p.add_argument("--min-delay", type=float, default=12.0, help="мин. пауза между карточками, с")
+    p.add_argument("--max-delay", type=float, default=30.0, help="макс. пауза между карточками, с")
+    p.add_argument("--render-min", type=float, default=6.0, help="мин. render-wait после goto, с")
+    p.add_argument("--render-max", type=float, default=10.0, help="макс. render-wait после goto, с")
+    p.add_argument("--limit-pages", type=int, default=100, help="макс. страниц на room-бакет")
+    p.add_argument("--enumerate-only", action="store_true", help="только перечислить, без detail")
+    p.add_argument(
+        "--headed",
+        action="store_true",
+        help="видимое окно браузера (debug); по умолчанию new-headless (окно скрыто)",
+    )
+    return p
+
+
+def main() -> None:
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(message)s",
+        datefmt="%H:%M:%S",
+    )
+    args = build_arg_parser().parse_args()
+    if args.max_delay < args.min_delay:
+        args.max_delay = args.min_delay
+    if args.render_max < args.render_min:
+        args.render_max = args.render_min
+
+    # Graceful Ctrl+C: asyncio.run превратит KeyboardInterrupt в CancelledError
+    # внутри run(); финальный flush — в finally.
+    try:
+        asyncio.run(run(args))
+    except KeyboardInterrupt:
+        logger.warning("KeyboardInterrupt at top level — exiting")
+
+
+if __name__ == "__main__":
+    main()
-- 
2.45.3


From 75f1d36505119c4a357056a94189eadb5f4ae1bc Mon Sep 17 00:00:00 2001
From: bot-backend 
Date: Sat, 27 Jun 2026 18:50:46 +0300
Subject: [PATCH 3/8] chore(tradein/domclick): configurable human-break
 (--break-min/--break-max, default 40-60s)

---
 tradein-mvp/backend/scripts/domclick_local_runner.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tradein-mvp/backend/scripts/domclick_local_runner.py b/tradein-mvp/backend/scripts/domclick_local_runner.py
index 12213417..742a285f 100644
--- a/tradein-mvp/backend/scripts/domclick_local_runner.py
+++ b/tradein-mvp/backend/scripts/domclick_local_runner.py
@@ -1132,7 +1132,7 @@ async def run(args: argparse.Namespace) -> None:
 
                 processed_since_break += 1
                 if processed_since_break >= next_break_at:
-                    brk = random.uniform(60, 180)
+                    brk = random.uniform(args.break_min, args.break_max)
                     logger.info(
                         "human break: sleeping %.0fs after %d cards", brk, processed_since_break
                     )
@@ -1160,6 +1160,10 @@ def build_arg_parser() -> argparse.ArgumentParser:
     p.add_argument("--max-delay", type=float, default=30.0, help="макс. пауза между карточками, с")
     p.add_argument("--render-min", type=float, default=6.0, help="мин. render-wait после goto, с")
     p.add_argument("--render-max", type=float, default=10.0, help="макс. render-wait после goto, с")
+    p.add_argument("--break-min", type=float, default=40.0, help="мин. длительность human-break, с")
+    p.add_argument(
+        "--break-max", type=float, default=60.0, help="макс. длительность human-break, с"
+    )
     p.add_argument("--limit-pages", type=int, default=100, help="макс. страниц на room-бакет")
     p.add_argument("--enumerate-only", action="store_true", help="только перечислить, без detail")
     p.add_argument(
-- 
2.45.3


From f7426dc12b4e74a89926a6667b3b6908cf2e6372 Mon Sep 17 00:00:00 2001
From: bot-backend 
Date: Sat, 27 Jun 2026 19:14:10 +0300
Subject: [PATCH 4/8] feat(tradein/domclick): JSONL ingest script for local
 detail-backfill runner
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

scripts/ingest_domclick_jsonl.py — idempotent ingest of domclick_local_runner
output: filter ok+price, upsert via save_listings (fault-tolerant house-match
hook in SAVEPOINT — non-matched listings still persist), deep-detail enrichment
via save_detail_enrichment (owners/encumbrances/kitchen/views/wall/floor/
price-history). --limit / --dry-run. psycopg v3 ANY(:ids) list-bind.
Per-record try/except in enrichment loop so one bad row never aborts the tail.

Live-tested in tradein-backend on 30 records: updated=30, house_matched=30,
detail_applied=30, price_history idempotent (75->75), enrich_failed=0.
---
 .../backend/scripts/ingest_domclick_jsonl.py  | 291 ++++++++++++++++++
 1 file changed, 291 insertions(+)
 create mode 100644 tradein-mvp/backend/scripts/ingest_domclick_jsonl.py

diff --git a/tradein-mvp/backend/scripts/ingest_domclick_jsonl.py b/tradein-mvp/backend/scripts/ingest_domclick_jsonl.py
new file mode 100644
index 00000000..a963818d
--- /dev/null
+++ b/tradein-mvp/backend/scripts/ingest_domclick_jsonl.py
@@ -0,0 +1,291 @@
+"""Ингест JSONL от локального DomClick-раннера (`scripts/domclick_local_runner.py`) в БД tradein.
+
+Запускается ВНУТРИ tradein-контейнера (импортит `app`). Идемпотентно.
+
+Каждая ok-запись (status=="ok" и валидный price_rub) →
+  1. upsert в `listings` через save_listings() (house-match хук — fault-tolerant, в SAVEPOINT);
+  2. deep-detail enrichment через save_detail_enrichment()
+     (owners / encumbrances / kitchen / views / wall / floor / price-history).
+
+Listing.id не возвращается из save_listings — достаём отдельным SELECT по (source, source_id).
+
+Usage:
+    python -m scripts.ingest_domclick_jsonl --jsonl /tmp/domclick.jsonl [--limit N] [--dry-run]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+from typing import Any
+
+from sqlalchemy import text
+
+from app.core.db import SessionLocal
+from app.services.scrapers.base import ScrapedLot, save_listings
+from app.services.scrapers.domclick_detail import (
+    DomClickDetailEnrichment,
+    save_detail_enrichment,
+)
+
+logger = logging.getLogger(__name__)
+
+SOURCE = "domklik"
+
+
+def _to_int(v: Any) -> int | None:
+    """int() None-safe: возвращает None на None / нечисловом мусоре (не крэшит прогон)."""
+    if v is None:
+        return None
+    try:
+        return int(v)
+    except (TypeError, ValueError):
+        return None
+
+
+def _to_float(v: Any) -> float | None:
+    """float() None-safe: возвращает None на None / нечисловом мусоре."""
+    if v is None:
+        return None
+    try:
+        return float(v)
+    except (TypeError, ValueError):
+        return None
+
+
+def _build_lot(rec: dict[str, Any]) -> ScrapedLot:
+    """Построить ScrapedLot из JSONL-записи. price_rub уже провалидирован caller'ом."""
+    raw_extra = rec.get("raw_extra") or {}
+    return ScrapedLot(
+        source=SOURCE,
+        source_url=rec["source_url"],
+        source_id=str(rec["id"]),
+        address=rec.get("address"),
+        lat=_to_float(rec.get("lat")),
+        lon=_to_float(rec.get("lon")),
+        rooms=_to_int(rec.get("rooms")),
+        area_m2=_to_float(rec.get("area_m2")),
+        floor=_to_int(rec.get("floor")),
+        total_floors=_to_int(rec.get("total_floors")),
+        year_built=_to_int(rec.get("year_built")),
+        price_rub=int(rec["price_rub"]),
+        listing_segment="vtorichka",
+        repair_state=rec.get("repair_state"),
+        raw_payload={
+            "is_sber_collateral": rec.get("is_sber_collateral"),
+            "square_price": raw_extra.get("square_price"),
+            "avm": rec.get("avm") or {},
+        },
+    )
+
+
+def _build_enrichment(rec: dict[str, Any]) -> DomClickDetailEnrichment:
+    """Построить deep-detail enrichment из JSONL-записи."""
+    raw_extra_in = rec.get("raw_extra") or {}
+    raw_extra: dict[str, Any] = {
+        "wall_type": rec.get("wall_type"),
+        "floor_type": rec.get("floor_type"),
+        "calls": rec.get("calls"),
+        "is_sber_collateral": rec.get("is_sber_collateral"),
+        "avm": rec.get("avm") or {},
+        **raw_extra_in,
+    }
+    return DomClickDetailEnrichment(
+        item_id=str(rec["id"]),
+        source_url=rec["source_url"],
+        repair_state=rec.get("repair_state"),
+        repair_type=rec.get("repair_raw"),
+        living_area_m2=_to_float(rec.get("living_area_m2")),
+        kitchen_area_m2=_to_float(rec.get("kitchen_area_m2")),
+        sale_type=rec.get("sale_type"),
+        owners_count=_to_int(rec.get("owners_count")),
+        encumbrances_clean=rec.get("encumbrances_clean"),
+        year_built=_to_int(rec.get("year_built")),
+        views_total=_to_int(rec.get("views")),
+        price_changes=rec.get("price_changes") or [],
+        raw_extra=raw_extra,
+    )
+
+
+def run(jsonl_path: str, limit: int | None, dry_run: bool) -> dict[str, int]:
+    """Прочитать JSONL, отфильтровать ok+price, upsert listings + enrichment.
+
+    Returns: dict со счётчиками для итогового лога / тестов.
+    """
+    lots: list[ScrapedLot] = []
+    valid_records: list[dict[str, Any]] = []  # (parallel to lots) для enrichment-шага
+
+    skipped_status = 0
+    skipped_no_price = 0
+    parse_failed = 0
+
+    with open(jsonl_path, encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError as exc:
+                logger.warning("битая JSON-строка пропущена: %s", exc)
+                parse_failed += 1
+                continue
+
+            if rec.get("status") != "ok":
+                skipped_status += 1
+                continue
+            price = _to_int(rec.get("price_rub"))
+            if price is None or price <= 0:
+                skipped_no_price += 1
+                continue
+
+            try:
+                lot = _build_lot(rec)
+            except Exception as exc:
+                logger.warning("не удалось построить lot для id=%s: %s", rec.get("id"), exc)
+                parse_failed += 1
+                continue
+
+            lots.append(lot)
+            valid_records.append(rec)
+
+            if limit is not None and len(lots) >= limit:
+                break
+
+    total_valid = len(lots)
+    logger.info(
+        "прочитано: valid=%d skipped_status=%d skipped_no_price=%d parse_failed=%d",
+        total_valid,
+        skipped_status,
+        skipped_no_price,
+        parse_failed,
+    )
+
+    inserted = 0
+    updated = 0
+    house_matched = 0
+    match_failed = 0
+    detail_applied = 0
+    price_history_attempted = 0
+    enrich_failed = 0
+
+    if dry_run:
+        logger.info("[dry-run] %d записей распарсено, запись в БД пропущена", total_valid)
+        return {
+            "inserted": inserted,
+            "updated": updated,
+            "house_matched": house_matched,
+            "match_failed": match_failed,
+            "detail_applied": detail_applied,
+            "price_history_attempted": price_history_attempted,
+            "enrich_failed": enrich_failed,
+            "skipped_status": skipped_status,
+            "skipped_no_price": skipped_no_price,
+            "parse_failed": parse_failed,
+        }
+
+    db = SessionLocal()
+    try:
+        if lots:
+            inserted, updated = save_listings(db, lots)
+            logger.info("save_listings: inserted=%d updated=%d", inserted, updated)
+
+        # Достаём listing.id + house_id_fk одним SELECT по source_id.
+        source_ids = [lot.source_id for lot in lots if lot.source_id is not None]
+        id_map: dict[str, tuple[int, Any]] = {}
+        if source_ids:
+            rows = db.execute(
+                text(
+                    "SELECT source_id, id, house_id_fk FROM listings "
+                    "WHERE source = :source AND source_id = ANY(:ids)"
+                ),
+                {"source": SOURCE, "ids": source_ids},
+            ).fetchall()
+            for sid, lid, house_id_fk in rows:
+                id_map[str(sid)] = (int(lid), house_id_fk)
+
+        total_saved = len(id_map)
+        house_matched = sum(1 for _, hid in id_map.values() if hid is not None)
+        match_failed = total_saved - house_matched
+
+        # Deep-detail enrichment по каждой ok-записи.
+        for rec in valid_records:
+            sid = str(rec["id"])
+            entry = id_map.get(sid)
+            if entry is None:
+                logger.warning("listing id не найден для source_id=%s — enrichment пропущен", sid)
+                continue
+            listing_id, _ = entry
+            # Per-record изоляция: save_detail_enrichment коммитит сама, поэтому
+            # падение одной записи иначе оборвало бы остаток valid_records трейсбэком.
+            # Уже закоммиченные строки персистят → повторный прогон добьёт хвост.
+            try:
+                enrichment = _build_enrichment(rec)
+                price_history_attempted += len(enrichment.price_changes)
+                if save_detail_enrichment(db, listing_id, enrichment):
+                    detail_applied += 1
+            except Exception as exc:
+                db.rollback()
+                enrich_failed += 1
+                logger.warning("enrichment упал для source_id=%s: %s", sid, exc)
+                continue
+    finally:
+        db.close()
+
+    logger.info(
+        "ИТОГО: saved(inserted=%d/updated=%d) house_matched=%d match_failed=%d "
+        "detail_applied=%d price_history_attempted=%d enrich_failed=%d "
+        "skipped_status=%d skipped_no_price=%d parse_failed=%d",
+        inserted,
+        updated,
+        house_matched,
+        match_failed,
+        detail_applied,
+        price_history_attempted,
+        enrich_failed,
+        skipped_status,
+        skipped_no_price,
+        parse_failed,
+    )
+
+    return {
+        "inserted": inserted,
+        "updated": updated,
+        "house_matched": house_matched,
+        "match_failed": match_failed,
+        "detail_applied": detail_applied,
+        "price_history_attempted": price_history_attempted,
+        "enrich_failed": enrich_failed,
+        "skipped_status": skipped_status,
+        "skipped_no_price": skipped_no_price,
+        "parse_failed": parse_failed,
+    }
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Ингест DomClick JSONL (от domclick_local_runner) в БД tradein.",
+    )
+    parser.add_argument("--jsonl", required=True, help="путь к JSONL-файлу раннера")
+    parser.add_argument(
+        "--limit",
+        type=int,
+        default=None,
+        help="обработать первые N валидных (ok+price) записей",
+    )
+    parser.add_argument(
+        "--dry-run",
+        action="store_true",
+        help="парсить + логировать, но НЕ писать в БД",
+    )
+    args = parser.parse_args()
+    run(args.jsonl, args.limit, args.dry_run)
+
+
+if __name__ == "__main__":
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(name)s: %(message)s",
+    )
+    main()
-- 
2.45.3


From 9b1f064069f80d6c7f56491cc033b42eb8dde70b Mon Sep 17 00:00:00 2001
From: bot-backend 
Date: Sat, 27 Jun 2026 20:03:05 +0300
Subject: [PATCH 5/8] =?UTF-8?q?feat(tradein/domclick-local):=20SERP-enumer?=
 =?UTF-8?q?ate=20(QRATOR-=D0=BE=D0=B1=D1=85=D0=BE=D0=B4)=20+=20offer-card?=
 =?UTF-8?q?=20v3=20extras?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

(а) enumerate: прямой bff-search offers/v1+count/v1 режет QRATOR-403 с home-IP →
переведён на навигацию по фронтовому SERP (page.goto, тот же чистый транспорт что
card-detail): id из DOM (a[href*=/card/]), total из , пагинация offset=20,
price-bisection при total>2000 ("5+"→5%2B через urlencode). Layer-A
(price/area/rooms/floor/total_floors/lat/lon/address) теперь из card-SSR
(productCard.priceInfo/objectInfo + address.position + houseInfo.info), т.к.
enumerate отдаёт только id+url. Удалены мёртвые BFF-функции. Схема JSONL не изменилась.

(б) detail: дозабор с offer-card v3 (200 с home-IP, в отличие от прод-прокси):
price_prediction (AVM заполненный + repair_quality + аренда, всегда) и sold_similar
(опц., --with-sold-similar/--sold-limit) через page.evaluate из QRATOR-сессии карточки.
XHR-AVM переопределяет пустой SSR pricePrediction; sold_similar в raw_extra.

Live-tested: enum-only собрал ids без QRATOR-аборта; full --limit 2 → price_rub
6.1M/3.1M, area/floor/coords/address из card-SSR, AVM market_price 4.99M/3.01M,
sold_similar.total≈1691. ruff clean.
---
 .../backend/scripts/domclick_local_runner.py  | 653 +++++++++++-------
 1 file changed, 399 insertions(+), 254 deletions(-)

diff --git a/tradein-mvp/backend/scripts/domclick_local_runner.py b/tradein-mvp/backend/scripts/domclick_local_runner.py
index 742a285f..de9542f8 100644
--- a/tradein-mvp/backend/scripts/domclick_local_runner.py
+++ b/tradein-mvp/backend/scripts/domclick_local_runner.py
@@ -14,28 +14,36 @@ DomClick фронтит QRATOR anti-bot. Проверено вживую (2026-0
 
 САМОДОСТАТОЧНОСТЬ
 =================
-Никаких ``app.*`` импортов: вся логика Layer A (BFF enumerate) и Layer B
+Никаких ``app.*`` импортов: вся логика Layer A (enumerate) и Layer B
 (detail ``__SSR_STATE__`` parse) скопирована ИНЛАЙН из
 ``app/services/scrapers/domclick.py`` + ``domclick_detail.py`` +
 ``repair_state_normalizer.py``. Зависимости: stdlib + ``playwright.async_api``.
 
 ТРАНСПОРТ
 =========
-Playwright Chromium, БЕЗ прокси (домашний IP):
-  • enumerate (BFF JSON) — ``context.request.get(url)`` (тот же сетевой
-    стек/IP/куки браузера) → парсим первый ``{`` … последний ``}``;
+Playwright Chromium, БЕЗ прокси (домашний IP). И enumerate, и detail идут
+ОДНИМ браузерным транспортом ``page.goto`` (verified 2026-06-27):
+  • enumerate — ``page.goto(SERP_url)`` на фронтовый
+    ``ekaterinburg.domclick.ru/search?...`` (SSR). Прямой BFF
+    ``bff-search-web.domclick.ru/api/offers/v1`` режет QRATOR-403 уже с
+    домашнего IP, а фронтовый SERP грузится чисто (тот же транспорт, что
+    card-detail). id офферов берём из DOM (``a[href*="/card/"]``), total —
+    из ``<title>`` ("… N объявлений"). Прямого offers/v1 XHR фронт НЕ шлёт.
+    Пагинация — параметром ``offset`` (page-size 20); при total>2000 —
+    price-bisection бакета (границы тоже из title-total).
   • detail — ``page.goto(card_url)`` → jittered wait → ``page.evaluate`` с
     ``fetch(location.href)`` чтобы получить СЫРОЙ HTML (SPA удаляет
     ``window.__SSR_STATE__`` после гидрации, поэтому нужен повторный fetch).
+    Layer-A (price/area/rooms/floor/total_floors/lat/lon/address) теперь
+    берётся ИЗ card-SSR (enumerate отдаёт только id+url).
 
 ВАЖНО — РЕЖИМ ОКНА (verified 2026-06-27 на домашнем IP этой машины)
 ------------------------------------------------------------------
 QRATOR ДЕТЕКТИТ СТАРЫЙ HEADLESS-РЕЖИМ: ``launch(headless=True)`` → даже с
-домашнего IP карточка отдаёт ``401 / 403 | Домклик`` (нет ``__SSR_STATE__``).
-А вот НОВЫЙ headless (``--headless=new``) И обычный headed → ``200`` + полный
-``__SSR_STATE__`` (~330 КБ). Поэтому по умолчанию запускаемся в new-headless
+домашнего IP карточка/SERP отдаёт ``401 / 403 | Домклик`` (нет
+``__SSR_STATE__``). А вот НОВЫЙ headless (``--headless=new``) И обычный headed →
+``200`` + полный рендер. Поэтому по умолчанию запускаемся в new-headless
 (окно НЕ видно, но детект проходит). ``--headed`` → видимое окно (debug).
-BFF-enumerate проходит в ЛЮБОМ режиме (это не браузерный рендер).
 
 ВАЖНО — ЧЕЛОВЕКОПОДОБНЫЙ ТЕМП (явное требование оператора)
 =========================================================
@@ -93,26 +101,26 @@ logger = logging.getLogger("domclick_local")
 
 # ── API / гео константы (ground truth, verified 2026-06-27) ──────────────────
 
-_BFF_BASE = "https://bff-search-web.domclick.ru"
-_EKB_ADDRESS_GUID = "0d475b79-88de-4054-818c-37d8f9d0d440"
-_EKB_AREA_ID = "20561"
-_CARD_HOST_FALLBACK = "https://ekaterinburg.domclick.ru"
+_EKB_AREA_ID = "20561"  # aids= фильтр ЕКБ-вторички на фронтовом SERP
+_SERP_HOST = "https://ekaterinburg.domclick.ru"
 _WARMUP_URL = "https://ekaterinburg.domclick.ru/"
 
-# Порядок бакетов комнат влияет на логи. urlencode("5+") → "5%2B".
+# Порядок бакетов комнат влияет на логи. КРИТИЧНО: "5+" обязан кодироваться как
+# "5%2B" — литеральный '+' в query фронт читает как пробел → фильтр игнорится и
+# отдаётся ПОЛНЫЙ свод (verified 2026-06-27). urlencode("5+") → "5%2B" — корректно.
+# "st" фронт принимает как студию. "5" в одиночку фронт НЕ понимает (только "5+").
 ROOM_BUCKETS: tuple[str, ...] = ("st", "1", "2", "3", "4", "5+")
 
-OFFSET_CAP: int = 2000  # макс. offset, принимаемый BFF
-PAGE_SIZE: int = 20  # items на страницу (жёстко задан API)
+OFFSET_CAP: int = 2000  # практический потолок offset фронтового SERP
+_SERP_PAGE_SIZE: int = 20  # офферов на страницу SERP (verified)
+# total из <title>: "... 2 269 объявлений" — число с разрядами (пробел/nbsp/thin).
+_SERP_TOTAL_RE = re.compile(r"(\d[\d\s]*)\s*объявлен")
 LTE_MAX: int = 1_000_000_000  # потолок цены для binary-split unbounded бакета
 MIN_PRICE_SPAN: int = 100_000  # ниже этого span деление прекращаем
 MAX_SPLIT_DEPTH: int = 30  # guard рекурсии binary-split (log2(1e9/1e5) ≈ 13)
 
-# Гео-guard ЕКБ (bbox). aids=20561 уже даёт чистый ЕКБ; bbox — defensive.
-# Deviation от app: ослаблено строгое равенство offerRegionName до bbox-only
-# (устойчивее к дрейфу имени региона; aids уже ограничивает выборку ЕКБ).
-_EKB_LAT_MIN, _EKB_LAT_MAX = 56.6, 57.0
-_EKB_LON_MIN, _EKB_LON_MAX = 60.2, 60.9
+# Гео-фильтр ЕКБ обеспечивает aids=20561 в SERP-URL; bbox-guard убран вместе с
+# BFF-enumerate (фронт отдаёт только id+url, координаты — из card-SSR).
 
 # Маркеры QRATOR / anti-bot challenge.
 _BLOCK_MARKERS: tuple[str, ...] = (
@@ -142,6 +150,17 @@ _FETCH_RAW_JS = (
     "{credentials: 'include'}); return await r.text(); }"
 )
 
+# JS: id+url всех офферов из DOM фронтового SERP (anchor'ы /card/...<8+ digits>).
+# Дедуп через Set по сериализованному объекту. Фронт — SSR, offers/v1 XHR НЕ шлёт.
+_SERP_IDS_JS = (
+    "() => [...new Set([...document.querySelectorAll('a[href*=\"/card/\"]')]"
+    ".map(a => { const h=a.getAttribute('href')||'';"
+    " const m=h.match(/\\/card\\/[a-z_]*?(\\d{8,})/);"
+    " return m ? {id:m[1], url: h.startsWith('http')? h :"
+    " 'https://ekaterinburg.domclick.ru'+h} : null; })"
+    ".filter(Boolean).map(o => JSON.stringify(o)))].map(s => JSON.parse(s))"
+)
+
 
 # ── Исключения ───────────────────────────────────────────────────────────────
 
@@ -360,7 +379,7 @@ def _extract_item_id(url: str) -> str | None:
     return m.group(1) if m else None
 
 
-# ── BFF JSON extraction (инлайн из domclick.py) ──────────────────────────────
+# ── Block detection / SERP URL & parsing ─────────────────────────────────────
 
 
 def _is_block_text(text: str) -> bool:
@@ -368,134 +387,55 @@ def _is_block_text(text: str) -> bool:
     return any(m in low for m in _BLOCK_MARKERS)
 
 
-def _extract_json(body: str) -> dict[str, Any]:
-    """Первый ``{`` … последний ``}`` из ответа BFF (raw JSON или <pre>-wrap).
-
-    Raises:
-        DomClickBlockedError: QRATOR/captcha маркеры в теле.
-        ValueError: JSON не найден / не dict.
-    """
-    if _is_block_text(body):
-        raise DomClickBlockedError("BFF: QRATOR block page detected")
-    start = body.find("{")
-    end = body.rfind("}")
-    if start == -1 or end == -1 or end <= start:
-        raise ValueError(f"No JSON object in BFF response (len={len(body)})")
-    data = json.loads(body[start : end + 1])
-    if not isinstance(data, dict):
-        raise ValueError(f"BFF JSON is not a dict: {type(data)}")
-    return data
-
-
-def _build_offers_url(rooms: str, gte: int | None, lte: int | None, offset: int) -> str:
+def _build_serp_url(rooms: str, gte: int | None, lte: int | None, offset: int) -> str:
+    """URL фронтового SERP. urlencode кодирует "5+" → "5%2B" (иначе фронт читает
+    '+' как пробел и игнорит room-фильтр)."""
     params: list[tuple[str, str]] = [
-        ("address", _EKB_ADDRESS_GUID),
-        ("aids", _EKB_AREA_ID),
         ("deal_type", "sale"),
         ("category", "living"),
         ("offer_type", "flat"),
-        ("rooms", rooms),
-        ("sort", "qi"),
-        ("sort_dir", "desc"),
+        ("aids", _EKB_AREA_ID),
+        ("from", "topline2020"),
         ("offset", str(offset)),
-        ("limit", str(PAGE_SIZE)),
-    ]
-    if gte is not None:
-        params.append(("sale_price__gte", str(gte)))
-    if lte is not None:
-        params.append(("sale_price__lte", str(lte)))
-    return f"{_BFF_BASE}/api/offers/v1?{urlencode(params)}"
-
-
-def _build_count_url(rooms: str, gte: int | None, lte: int | None) -> str:
-    params: list[tuple[str, str]] = [
-        ("address", _EKB_ADDRESS_GUID),
-        ("aids", _EKB_AREA_ID),
-        ("deal_type", "sale"),
-        ("category", "living"),
-        ("offer_type", "flat"),
         ("rooms", rooms),
-        ("sort", "qi"),
-        ("sort_dir", "desc"),
     ]
     if gte is not None:
         params.append(("sale_price__gte", str(gte)))
     if lte is not None:
         params.append(("sale_price__lte", str(lte)))
-    return f"{_BFF_BASE}/api/offers/count/v1?{urlencode(params)}"
+    return f"{_SERP_HOST}/search?{urlencode(params)}"
 
 
-def _is_geo_ok(item: dict[str, Any]) -> bool:
-    """bbox-guard ЕКБ. lat/lon обязательны и в пределах bbox."""
-    loc = item.get("location") or {}
-    lat = loc.get("lat")
-    lon = loc.get("lon")
-    if lat is None or lon is None:
-        return False
-    try:
-        latf, lonf = float(lat), float(lon)
-    except (ValueError, TypeError):
-        return False
-    return _EKB_LAT_MIN <= latf <= _EKB_LAT_MAX and _EKB_LON_MIN <= lonf <= _EKB_LON_MAX
-
-
-# ── SERP item → плоский dict (инлайн из domclick.py _map_item) ────────────────
-
-
-def map_serp_item(item: dict[str, Any], force_rooms: int | None) -> dict[str, Any] | None:
-    """offer-item из BFF → SERP-dict. None при price<=0 / без id / ошибке."""
-    try:
-        price = item.get("price") or 0
-        if price <= 0:
-            return None
-        item_id = item.get("id")
-        if item_id is None:
-            return None
-        sid = str(item_id)
-
-        path = item.get("path") or ""
-        if path.startswith("/"):
-            path = _CARD_HOST_FALLBACK + path
-
-        loc = item.get("location") or {}
-        oi = item.get("objectInfo") or {}
-        house = item.get("house") or {}
-        addr = item.get("address") or {}
-
-        area_raw = oi.get("area")
-        floor_raw = oi.get("floor")
-        floors_raw = house.get("floors")
-        year_raw = house.get("buildYear")
-        sq_raw = item.get("squarePrice")
-
-        if force_rooms is not None:
-            rooms: int | None = force_rooms
-        else:
-            rr = oi.get("rooms")
-            rooms = int(rr) if rr is not None else None
-
-        return {
-            "id": sid,
-            "source_url": path,
-            "price_rub": int(price),
-            "square_price": int(sq_raw) if sq_raw else None,
-            "area_m2": float(area_raw) if area_raw else None,
-            "rooms": rooms,
-            "floor": int(floor_raw) if floor_raw else None,
-            "total_floors": int(floors_raw) if floors_raw else None,
-            "year_built": int(year_raw) if year_raw else None,
-            "lat": loc.get("lat"),
-            "lon": loc.get("lon"),
-            "address": addr.get("displayName"),
-            "is_apartment": oi.get("isApartment"),
-            "offer_region_name": item.get("offerRegionName"),
-            "description": item.get("description"),
-            "is_sber_collateral": item.get("isSberCollateral"),
-            "published_date": item.get("publishedDate"),
-        }
-    except Exception:
-        logger.warning("map_serp_item failed id=%r", item.get("id"), exc_info=True)
+def _parse_serp_total(title: str) -> int | None:
+    """total из <title> ("… 2 269 объявлений"). None если числа нет."""
+    if not title:
         return None
+    m = _SERP_TOTAL_RE.search(title)
+    if not m:
+        return None
+    digits = re.sub(r"\D", "", m.group(1))
+    return int(digits) if digits else None
+
+
+def _canon_card_url(sid: str, url: str | None) -> str:
+    """Канонизируем source_url к /card/sale__flat__<id>, если href неполный."""
+    if url and url.startswith("http") and "/card/" in url:
+        return url
+    return f"{_SERP_HOST}/card/sale__flat__{sid}"
+
+
+def _serp_offer_to_dict(raw: Any) -> dict[str, Any] | None:
+    """offer из _SERP_IDS_JS ({id,url}) → минимальный SERP-dict {id, source_url}.
+    Layer-A (price/area/...) теперь добирается из card-SSR в parse_detail_html."""
+    if not isinstance(raw, dict):
+        return None
+    sid = raw.get("id")
+    if sid is None:
+        return None
+    sid = str(sid)
+    if not sid.isdigit():
+        return None
+    return {"id": sid, "source_url": _canon_card_url(sid, raw.get("url"))}
 
 
 # ── SSR state extraction (инлайн из domclick_detail.py) ──────────────────────
@@ -601,6 +541,27 @@ def parse_detail_html(html: str, source_url: str) -> dict[str, Any]:
         views = _to_int(pc.get("viewsCount"))
         calls = _to_int(pc.get("callsCount"))
 
+        # Layer-A из card-SSR (раньше шёл из BFF serp; теперь enumerate даёт
+        # только id+url). Пути verified 2026-06-27: цена/squarePrice — в
+        # priceInfo; area/rooms/floor — в objectInfo; total_floors — houseInfo;
+        # lat/lon/address — productCard.address(.position).
+        price_info = pc.get("priceInfo") or {}
+        address = pc.get("address") or {}
+        position = address.get("position") or {}
+        price_rub = _to_int(price_info.get("price"))
+        square_price = _to_int(price_info.get("squarePrice"))
+        area_m2 = _pos_float(oi.get("area"))
+        rooms = _to_int(oi.get("rooms"))
+        floor = _to_int(oi.get("floor"))
+        total_floors = _to_int(hi.get("floors")) or _to_int(
+            (pc.get("house") or {}).get("info", {}).get("floors")
+        )
+        lat = _to_float(position.get("lat"))
+        lon = _to_float(position.get("lon"))
+        address_name = address.get("displayName")
+        is_apartment = oi.get("isApartment")
+        description = oi.get("description")
+
         # priceHistory → нормализованные записи (skip при битой дате/без цены).
         raw_history = (pc.get("priceInfo") or {}).get("priceHistory") or []
         price_changes: list[dict[str, Any]] = []
@@ -664,6 +625,19 @@ def parse_detail_html(html: str, source_url: str) -> dict[str, Any]:
         )
 
         return {
+            # Layer-A (из card-SSR; раньше из BFF serp).
+            "price_rub": price_rub,
+            "square_price": square_price,
+            "area_m2": area_m2,
+            "rooms": rooms,
+            "floor": floor,
+            "total_floors": total_floors,
+            "lat": lat,
+            "lon": lon,
+            "address": address_name,
+            "is_apartment": is_apartment,
+            "description": description,
+            # Layer-B.
             "repair_state": repair_state,
             "repair_raw": repair_raw,
             "living_area_m2": living,
@@ -698,27 +672,29 @@ def build_record(
     d = detail or {}
     repair_state = d.get("repair_state")
     if not repair_state:
-        # fallback-инференс из SERP-описания (как в проде для пустого структурного)
-        repair_state = infer_repair_state_from_text(serp.get("description"))
+        # fallback-инференс из описания (теперь из card-SSR; serp-описания больше нет)
+        repair_state = infer_repair_state_from_text(d.get("description") or serp.get("description"))
 
     is_sber = serp.get("is_sber_collateral")
     if not is_sber and d.get("collateral_sber"):
         is_sber = True
 
+    # Layer-A: приоритет card-SSR (d), фолбэк на serp (на случай если serp когда-то
+    # снова начнёт отдавать поля). enumerate теперь даёт только id+source_url.
     return {
         "id": serp.get("id"),
         "source_url": serp.get("source_url"),
         "status": status,
         "fetched_at": datetime.now(UTC).isoformat(),
-        "price_rub": serp.get("price_rub"),
-        "area_m2": serp.get("area_m2"),
-        "rooms": serp.get("rooms"),
-        "floor": serp.get("floor"),
-        "total_floors": serp.get("total_floors"),
+        "price_rub": d.get("price_rub") or serp.get("price_rub"),
+        "area_m2": d.get("area_m2") or serp.get("area_m2"),
+        "rooms": d.get("rooms") if d.get("rooms") is not None else serp.get("rooms"),
+        "floor": d.get("floor") or serp.get("floor"),
+        "total_floors": d.get("total_floors") or serp.get("total_floors"),
         "year_built": d.get("year_built") or serp.get("year_built"),
-        "lat": serp.get("lat"),
-        "lon": serp.get("lon"),
-        "address": serp.get("address"),
+        "lat": d.get("lat") or serp.get("lat"),
+        "lon": d.get("lon") or serp.get("lon"),
+        "address": d.get("address") or serp.get("address"),
         "repair_state": repair_state,
         "repair_raw": d.get("repair_raw"),
         "living_area_m2": d.get("living_area_m2"),
@@ -734,11 +710,16 @@ def build_record(
         "is_sber_collateral": is_sber,
         "avm": d.get("avm") or {},
         "raw_extra": {
-            "square_price": serp.get("square_price"),
-            "is_apartment": serp.get("is_apartment"),
+            "square_price": d.get("square_price") or serp.get("square_price"),
+            "is_apartment": (
+                d.get("is_apartment")
+                if d.get("is_apartment") is not None
+                else serp.get("is_apartment")
+            ),
             "offer_region_name": serp.get("offer_region_name"),
             "published_date": serp.get("published_date"),
             **(d.get("raw_extra") or {}),
+            **({"sold_similar": d["sold_similar"]} if d.get("sold_similar") else {}),
         },
     }
 
@@ -774,79 +755,103 @@ def append_record(path: Path, rec: dict[str, Any]) -> None:
         fh.flush()
 
 
-# ── BFF transport (playwright APIRequestContext) ─────────────────────────────
+# ── SERP transport (playwright page navigation, чистый браузерный транспорт) ──
 
 
-async def fetch_bff_json(context: Any, url: str) -> dict[str, Any]:
-    """GET BFF через context.request (тот же IP/куки браузера) → dict."""
-    resp = await context.request.get(
-        url,
-        headers={
-            "Accept": "application/json, text/plain, */*",
-            "Referer": "https://ekaterinburg.domclick.ru/search?deal_type=sale",
-            "Accept-Language": "ru-RU,ru;q=0.9",
-        },
-        timeout=60000,
-    )
-    body = await resp.text()
-    if resp.status == 403:
-        raise DomClickBlockedError(f"BFF 403 for {url}")
-    return _extract_json(body)
+async def fetch_serp_page(
+    page: Any,
+    url: str,
+    render_wait: tuple[float, float],
+) -> tuple[list[dict[str, Any]], int | None, int]:
+    """Навигация на фронтовый SERP → (offers, total, http_status).
+
+    offers — список {id, source_url} из DOM; total — из <title>.
+    Raises:
+        DomClickBlockedError: 403 или QRATOR-маркеры в title.
+    """
+    resp = await page.goto(url, wait_until="domcontentloaded", timeout=70000)
+    await asyncio.sleep(random.uniform(*render_wait))  # дать SSR отрендериться
+    status = resp.status if resp else 0
+    title = ""
+    try:
+        title = (await page.title()) or ""
+    except Exception:
+        title = ""
+    if status == 403 or _is_block_text(title):
+        raise DomClickBlockedError(f"SERP block status={status} title={title!r}")
+    raw = await page.evaluate(_SERP_IDS_JS)
+    offers: list[dict[str, Any]] = []
+    if isinstance(raw, list):
+        for item in raw:
+            serp = _serp_offer_to_dict(item)
+            if serp is not None:
+                offers.append(serp)
+    total = _parse_serp_total(title)
+    return offers, total, status
 
 
-async def get_count(context: Any, rooms: str, gte: int | None, lte: int | None) -> int:
-    data = await fetch_bff_json(context, _build_count_url(rooms, gte, lte))
-    res = data.get("result") or {}
-    raw = res.get("snippetsCount")
-    return int(raw) if raw else 0
+# ── Enumerate (mirror domclick.py _sweep_bucket / _paginate, на title-total) ──
 
 
-# ── Enumerate (mirror domclick.py _sweep_bucket / _paginate) ─────────────────
+def _add_offers(
+    offers: list[dict[str, Any]],
+    seen: set[str],
+    done: set[str],
+    out: list[dict[str, Any]],
+    target_new: int | None,
+) -> int:
+    """Добавить новые офферы в out (dedup по seen/done). → число добавленных.
+    Бросает _EnumLimitReachedError при достижении target_new."""
+    added = 0
+    for serp in offers:
+        sid = serp["id"]
+        if sid in seen or sid in done:
+            continue
+        seen.add(sid)
+        out.append(serp)
+        added += 1
+        if target_new is not None and len(out) >= target_new:
+            raise _EnumLimitReachedError
+    return added
 
 
 async def _paginate(
-    context: Any,
+    page: Any,
     rooms: str,
     gte: int | None,
     lte: int | None,
+    total: int | None,
     seen: set[str],
     done: set[str],
     out: list[dict[str, Any]],
     limit_pages: int,
     target_new: int | None,
+    render_wait: tuple[float, float],
     enum_jitter: tuple[float, float],
+    start_page: int,
 ) -> None:
-    force_rooms = 0 if rooms == "st" else None
-    page_idx = 0
+    """Пагинация SERP с offset=start_page*20 далее. Стоп: достигли total/cap,
+    кончились limit_pages, или страница не дала новых id."""
+    bound = OFFSET_CAP if total is None else min(total, OFFSET_CAP)
+    page_idx = start_page
     while page_idx < limit_pages:
-        offset = page_idx * PAGE_SIZE
-        if offset >= OFFSET_CAP:
-            break
-        data = await fetch_bff_json(context, _build_offers_url(rooms, gte, lte, offset))
-        items = (data.get("result") or {}).get("items") or []
-        if not items:
-            break
-        for item in items:
-            serp = map_serp_item(item, force_rooms)
-            if serp is None:
-                continue
-            if not _is_geo_ok(item):
-                continue
-            sid = serp["id"]
-            if sid in seen or sid in done:
-                continue
-            seen.add(sid)
-            out.append(serp)
-            if target_new is not None and len(out) >= target_new:
-                raise _EnumLimitReachedError
-        page_idx += 1
-        if len(items) < PAGE_SIZE:
+        offset = page_idx * _SERP_PAGE_SIZE
+        if offset >= bound:
             break
         await asyncio.sleep(random.uniform(*enum_jitter))
+        offers, _, _ = await fetch_serp_page(
+            page, _build_serp_url(rooms, gte, lte, offset), render_wait
+        )
+        if not offers:
+            break
+        added = _add_offers(offers, seen, done, out, target_new)
+        page_idx += 1
+        if added == 0:
+            break
 
 
 async def _sweep_bucket(
-    context: Any,
+    page: Any,
     rooms: str,
     gte: int | None,
     lte: int | None,
@@ -855,86 +860,120 @@ async def _sweep_bucket(
     out: list[dict[str, Any]],
     limit_pages: int,
     target_new: int | None,
+    render_wait: tuple[float, float],
     enum_jitter: tuple[float, float],
     depth: int,
 ) -> None:
-    count = await get_count(context, rooms, gte, lte)
-    if count == 0:
-        return
-    if count > OFFSET_CAP and depth < MAX_SPLIT_DEPTH:
-        _gte = gte if gte is not None else 0
-        _lte = lte if lte is not None else LTE_MAX
-        span = _lte - _gte
-        if span <= MIN_PRICE_SPAN:
-            logger.warning(
-                "rooms=%r price=[%s,%s] count=%d > cap but span<=%d — paginating as-is",
+    """offset=0 → total + page-0 офферы. total>cap → price-bisection, иначе
+    пагинируем остаток. total читаем из <title> (чистый фронтовый транспорт)."""
+    offers0, total, _ = await fetch_serp_page(
+        page, _build_serp_url(rooms, gte, lte, 0), render_wait
+    )
+    _add_offers(offers0, seen, done, out, target_new)
+    if not total:
+        # нет числа / 0 объявлений: офферов на page-0 нет → выходим; есть → допагинируем.
+        if offers0:
+            await _paginate(
+                page,
                 rooms,
                 gte,
                 lte,
-                count,
-                MIN_PRICE_SPAN,
+                None,
+                seen,
+                done,
+                out,
+                limit_pages,
+                target_new,
+                render_wait,
+                enum_jitter,
+                1,
             )
-            await _paginate(
-                context, rooms, gte, lte, seen, done, out, limit_pages, target_new, enum_jitter
+        return
+    if total > OFFSET_CAP and depth < MAX_SPLIT_DEPTH:
+        _gte = gte if gte is not None else 0
+        _lte = lte if lte is not None else LTE_MAX
+        span = _lte - _gte
+        if span > MIN_PRICE_SPAN:
+            mid = (_gte + _lte) // 2
+            logger.info(
+                "rooms=%r total=%d > cap — split [%d,%d]+[%d,%d]",
+                rooms,
+                total,
+                _gte,
+                mid,
+                mid + 1,
+                _lte,
+            )
+            await _sweep_bucket(
+                page,
+                rooms,
+                _gte,
+                mid,
+                seen,
+                done,
+                out,
+                limit_pages,
+                target_new,
+                render_wait,
+                enum_jitter,
+                depth + 1,
+            )
+            await _sweep_bucket(
+                page,
+                rooms,
+                mid + 1,
+                _lte,
+                seen,
+                done,
+                out,
+                limit_pages,
+                target_new,
+                render_wait,
+                enum_jitter,
+                depth + 1,
             )
             return
-        mid = (_gte + _lte) // 2
-        logger.info(
-            "rooms=%r count=%d > cap — split [%d,%d]+[%d,%d]",
+        logger.warning(
+            "rooms=%r price=[%s,%s] total=%d > cap но span<=%d — пагинируем как есть",
             rooms,
-            count,
-            _gte,
-            mid,
-            mid + 1,
-            _lte,
-        )
-        await _sweep_bucket(
-            context,
-            rooms,
-            _gte,
-            mid,
-            seen,
-            done,
-            out,
-            limit_pages,
-            target_new,
-            enum_jitter,
-            depth + 1,
-        )
-        await _sweep_bucket(
-            context,
-            rooms,
-            mid + 1,
-            _lte,
-            seen,
-            done,
-            out,
-            limit_pages,
-            target_new,
-            enum_jitter,
-            depth + 1,
-        )
-    else:
-        await _paginate(
-            context, rooms, gte, lte, seen, done, out, limit_pages, target_new, enum_jitter
+            gte,
+            lte,
+            total,
+            MIN_PRICE_SPAN,
         )
+    await _paginate(
+        page,
+        rooms,
+        gte,
+        lte,
+        total,
+        seen,
+        done,
+        out,
+        limit_pages,
+        target_new,
+        render_wait,
+        enum_jitter,
+        1,
+    )
 
 
 async def enumerate_cards(
-    context: Any,
+    page: Any,
     done: set[str],
     limit_pages: int,
     target_new: int | None,
+    render_wait: tuple[float, float],
     enum_jitter: tuple[float, float],
 ) -> list[dict[str, Any]]:
-    """Свод ЕКБ-вторички через BFF: все room-бакеты, dedup, skip done."""
+    """Свод ЕКБ-вторички через фронтовый SERP: все room-бакеты, dedup, skip done."""
     out: list[dict[str, Any]] = []
     seen: set[str] = set()
     try:
         for bucket in ROOM_BUCKETS:
             logger.info("enumerate: bucket rooms=%r (collected=%d)", bucket, len(out))
             await _sweep_bucket(
-                context,
+                page,
                 bucket,
                 None,
                 None,
@@ -943,6 +982,7 @@ async def enumerate_cards(
                 out,
                 limit_pages,
                 target_new,
+                render_wait,
                 enum_jitter,
                 0,
             )
@@ -975,10 +1015,90 @@ async def fetch_card_html(
     return html, title, status
 
 
+# JS: дозабор offer-card v3 (AVM price_prediction + опц. sold_similar) из
+# контекста УЖЕ загруженной страницы — та же сессия/куки, что прошли QRATOR.
+_OFFER_CARD_EXTRAS_JS = """async (args) => {
+  const [id, withSold, soldLimit] = args;
+  const out = {};
+  const get = async (u) => {
+    try {
+      const r = await fetch(u, {credentials: 'include'});
+      return r.ok ? await r.json() : {__status: r.status};
+    } catch (e) {
+      return {__err: String(e)};
+    }
+  };
+  out.price_prediction = await get(
+    'https://offer-card.domclick.ru/api/v3/offers/price_prediction?offer_id=' + id
+  );
+  if (withSold) {
+    out.sold_similar = await get(
+      'https://offer-card.domclick.ru/api/v3/offers/sold_similar/flat/' + id +
+      '?in_house=false&limit=' + soldLimit + '&offset=0'
+    );
+  }
+  return out;
+}"""
+
+
+def _parse_offer_card_avm(price_prediction: Any) -> dict[str, Any] | None:
+    """price_prediction.result → AVM-форма (как d["avm"] из SSR). None если пусто."""
+    if not isinstance(price_prediction, dict):
+        return None
+    result = price_prediction.get("result")
+    if not isinstance(result, dict):
+        return None
+    market_price = result.get("market_price")
+    if market_price is None:
+        return None
+    avm = {
+        "market_price": market_price,
+        "min": result.get("min_market_price"),
+        "max": result.get("max_market_price"),
+        "repair_quality": result.get("repair_quality"),
+        "rent_long": result.get("rent_long"),
+        "rent_short": result.get("rent_short"),
+    }
+    return {k: v for k, v in avm.items() if v is not None}
+
+
+def _parse_offer_card_sold(sold_similar: Any) -> dict[str, Any] | None:
+    """sold_similar → {total, items}. None если нет result.items."""
+    if not isinstance(sold_similar, dict):
+        return None
+    result = sold_similar.get("result")
+    if not isinstance(result, dict):
+        return None
+    items = result.get("items")
+    if not isinstance(items, list):
+        return None
+    pagination = sold_similar.get("pagination") or {}
+    total = pagination.get("total") if isinstance(pagination, dict) else None
+    return {"total": total, "items": items}
+
+
+async def fetch_offer_card_extras(
+    page: Any,
+    item_id: str,
+    with_sold_similar: bool,
+    sold_limit: int,
+) -> dict[str, Any]:
+    """Дозабор AVM (всегда) + sold_similar (опц.) с offer-card v3 из контекста
+    загруженной страницы. None-safe: любая ошибка/__status/__err → поле None."""
+    raw = await page.evaluate(_OFFER_CARD_EXTRAS_JS, [str(item_id), with_sold_similar, sold_limit])
+    if not isinstance(raw, dict):
+        return {"avm": None, "sold_similar": None}
+    avm = _parse_offer_card_avm(raw.get("price_prediction"))
+    sold = _parse_offer_card_sold(raw.get("sold_similar")) if with_sold_similar else None
+    return {"avm": avm, "sold_similar": sold}
+
+
 async def process_card(
     page: Any,
     serp: dict[str, Any],
     render_wait: tuple[float, float],
+    with_sold_similar: bool,
+    sold_limit: int,
 ) -> tuple[str, dict[str, Any] | None]:
     """Один detail-fetch с одним ретраем на блок. → (status, detail|None)."""
     url = serp["source_url"]
@@ -988,6 +1108,16 @@ async def process_card(
             if http_status == 403 or _is_block_text(title) or _is_block_text(html[:4096]):
                 raise DomClickBlockedError(f"403/challenge title={title!r}")
             detail = parse_detail_html(html, url)
+            try:
+                extras = await fetch_offer_card_extras(
+                    page, serp["id"], with_sold_similar, sold_limit
+                )
+                if extras.get("avm"):
+                    detail["avm"] = extras["avm"]
+                if extras.get("sold_similar"):
+                    detail["sold_similar"] = extras["sold_similar"]
+            except Exception as exc:
+                logger.warning("card %s offer-card extras failed: %s", serp["id"], exc)
             return "ok", detail
         except DomClickBlockedError as exc:
             if attempt == 1:
@@ -1049,6 +1179,9 @@ async def run(args: argparse.Namespace) -> None:
     summary = Summary()
     render_wait = (args.render_min, args.render_max)
     target_new = args.limit if args.limit and args.limit > 0 else None
+    # SERP-навигация требует render-wait на SSR; enum_jitter — доп. пейсинг между
+    # страницами (SERP-навигаций много при свипе, бережём home-IP).
+    enum_render = (max(2.0, args.render_min - 2.0), max(3.0, args.render_max - 2.0))
     enum_jitter = (1.5, 4.0)
 
     # ВАЖНО: launch(headless=True) = СТАРЫЙ headless → QRATOR 403 даже с дом. IP.
@@ -1069,7 +1202,7 @@ async def run(args: argparse.Namespace) -> None:
         )
         page = await context.new_page()
 
-        # Warm-up: устанавливаем QRATOR-куки .domclick.ru перед BFF-запросами.
+        # Warm-up: устанавливаем QRATOR-куки .domclick.ru перед SERP-навигацией.
         try:
             await page.goto(_WARMUP_URL, wait_until="domcontentloaded", timeout=70000)
             await asyncio.sleep(random.uniform(2.0, 4.0))
@@ -1077,8 +1210,10 @@ async def run(args: argparse.Namespace) -> None:
         except Exception as exc:
             logger.warning("warmup navigation failed (%s) — continuing", exc)
 
-        # 1. Enumerate.
-        cards = await enumerate_cards(context, done, args.limit_pages, target_new, enum_jitter)
+        # 1. Enumerate через фронтовый SERP (тот же чистый браузерный транспорт).
+        cards = await enumerate_cards(
+            page, done, args.limit_pages, target_new, enum_render, enum_jitter
+        )
         summary.enumerated = len(cards)
 
         if args.enumerate_only:
@@ -1102,7 +1237,9 @@ async def run(args: argparse.Namespace) -> None:
                     summary.skipped_done += 1
                     continue
 
-                status, detail = await process_card(page, serp, render_wait)
+                status, detail = await process_card(
+                    page, serp, render_wait, args.with_sold_similar, args.sold_limit
+                )
                 rec = build_record(serp, detail, status)
                 append_record(out_path, rec)
                 done.add(sid)
@@ -1166,6 +1303,14 @@ def build_arg_parser() -> argparse.ArgumentParser:
     )
     p.add_argument("--limit-pages", type=int, default=100, help="макс. страниц на room-бакет")
     p.add_argument("--enumerate-only", action="store_true", help="только перечислить, без detail")
+    p.add_argument(
+        "--with-sold-similar",
+        action="store_true",
+        help="дозабор sold_similar с offer-card v3 (+1 запрос/карточку)",
+    )
+    p.add_argument(
+        "--sold-limit", type=int, default=5, help="limit для sold_similar (если включён)"
+    )
     p.add_argument(
         "--headed",
         action="store_true",
-- 
2.45.3


From 3cc015c2493e2b0356d848e9cb952b74dcc5f94c Mon Sep 17 00:00:00 2001
From: bot-backend <bot-backend@gendsgn.local>
Date: Sat, 27 Jun 2026 22:26:48 +0300
Subject: [PATCH 6/8] =?UTF-8?q?fix(tradein/domclick-local):=20SERP=20enume?=
 =?UTF-8?q?rate=20=E2=80=94=20wait=5Ffor=5Fselector=20=D0=B2=D0=BC=D0=B5?=
 =?UTF-8?q?=D1=81=D1=82=D0=BE=20=D1=81=D0=BB=D0=B5=D0=BF=D0=BE=D0=B3=D0=BE?=
 =?UTF-8?q?=20render-wait?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Карточки SERP появляются клиентской гидрацией через ~6-12с после domcontentloaded.
Фиксированный render-sleep ловил пустой DOM при медленном ответе → enumerate молча
собирал 0 ids (silent 0-coverage; всплыло при ускоренном профиле render 3-5).
Теперь fetch_serp_page ждёт 'a[href*=/card/]' явно (timeout 12с) → пустой ценовой
band корректно даёт 0 по таймауту, render_wait остаётся коротким settle. Подтверждено
enumerate-only --limit 5 с render 3-5.
---
 tradein-mvp/backend/scripts/domclick_local_runner.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/tradein-mvp/backend/scripts/domclick_local_runner.py b/tradein-mvp/backend/scripts/domclick_local_runner.py
index de9542f8..26e7ff9e 100644
--- a/tradein-mvp/backend/scripts/domclick_local_runner.py
+++ b/tradein-mvp/backend/scripts/domclick_local_runner.py
@@ -770,7 +770,15 @@ async def fetch_serp_page(
         DomClickBlockedError: 403 или QRATOR-маркеры в title.
     """
     resp = await page.goto(url, wait_until="domcontentloaded", timeout=70000)
-    await asyncio.sleep(random.uniform(*render_wait))  # дать SSR отрендериться
+    # SERP рендерит карточки клиентской гидрацией через ~6-12с после domcontentloaded.
+    # Фиксированный sleep хрупок: при медленном ответе DOM ещё пуст → 0 ids (silent
+    # 0-coverage). Ждём появления card-ссылок ЯВНО; genuinely-пустой ценовой band →
+    # timeout → 0 offers (корректно). render_wait остаётся коротким settle поверх.
+    try:
+        await page.wait_for_selector("a[href*='/card/']", timeout=12000)
+    except Exception:
+        pass  # пустой band или медленный рендер — читаем как есть (0 offers)
+    await asyncio.sleep(random.uniform(*render_wait))  # settle: дорисовать все 20 + title
     status = resp.status if resp else 0
     title = ""
     try:
-- 
2.45.3


From 0f7f5ea0ab0e61e49963f2f66ca9b5d1b5064b81 Mon Sep 17 00:00:00 2001
From: bot-backend <bot-backend@gendsgn.local>
Date: Sat, 27 Jun 2026 22:39:47 +0300
Subject: [PATCH 7/8] =?UTF-8?q?fix(tradein/domclick-local):=20resume=20?=
 =?UTF-8?q?=E2=80=94=20=D1=83=D0=B1=D1=80=D0=B0=D1=82=D1=8C=20=D0=BF=D1=80?=
 =?UTF-8?q?=D0=B5=D0=B6=D0=B4=D0=B5=D0=B2=D1=80=D0=B5=D0=BC=D0=B5=D0=BD?=
 =?UTF-8?q?=D0=BD=D1=8B=D0=B9=20added=3D=3D0=20break=20=D0=B2=20=5Fpaginat?=
 =?UTF-8?q?e?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

При resume первые страницы каждого бакета = топ-qi офферы, уже лежащие в done →
_add_offers возвращал added=0 → _paginate делал break → enumerate собирал 0 новых
(st=0 при resume; на первом прогоне done пуст, не проявлялось). Терминаторы пагинации
— только пустая страница и offset>=bound. Подтверждено: resume со 114 done → st собрал
546 новых студий (660-114).
---
 tradein-mvp/backend/scripts/domclick_local_runner.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tradein-mvp/backend/scripts/domclick_local_runner.py b/tradein-mvp/backend/scripts/domclick_local_runner.py
index 26e7ff9e..8f7979fd 100644
--- a/tradein-mvp/backend/scripts/domclick_local_runner.py
+++ b/tradein-mvp/backend/scripts/domclick_local_runner.py
@@ -852,10 +852,11 @@ async def _paginate(
         )
         if not offers:
             break
-        added = _add_offers(offers, seen, done, out, target_new)
+        # NB: НЕ break при added==0. При resume первые страницы бакета — это топ-qi
+        # офферы, уже лежащие в `done` → added=0, но новые id есть на след. страницах.
+        # Терминаторы — только пустая страница (выше) и offset>=bound (total/cap).
+        _add_offers(offers, seen, done, out, target_new)
         page_idx += 1
-        if added == 0:
-            break
 
 
 async def _sweep_bucket(
-- 
2.45.3


From 5f994cc3009a957f3d150c1c49396a8a4b1b6deb Mon Sep 17 00:00:00 2001
From: bot-backend <bot-backend@gendsgn.local>
Date: Sat, 27 Jun 2026 23:37:07 +0300
Subject: [PATCH 8/8] =?UTF-8?q?feat(tradein/domclick-local):=20--enum-cach?=
 =?UTF-8?q?e=20=E2=80=94=20enumerate=20=D0=BE=D0=B4=D0=B8=D0=BD=20=D1=80?=
 =?UTF-8?q?=D0=B0=D0=B7,=20resume=20detail=20=D0=BF=D0=BE=D0=B2=D0=B5?=
 =?UTF-8?q?=D1=80=D1=85?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

enumerate (~6422 id через SERP, ~1.5-2ч) теперь пишет полный список id в файл-кэш
(--enum-cache PATH) с sentinel завершённости; следующий запуск грузит кэш и полностью
пропускает enumerate → сразу detail. detail-resume по done (JSONL) композится поверх.
Кэш пишется только после реально завершённого свода (прерванный enumerate не кэшируется).
Тест: run1 записал 5 id+sentinel, run2 загрузил и пропустил SERP.
---
 .../backend/scripts/domclick_local_runner.py  | 84 ++++++++++++++++++-
 1 file changed, 81 insertions(+), 3 deletions(-)

diff --git a/tradein-mvp/backend/scripts/domclick_local_runner.py b/tradein-mvp/backend/scripts/domclick_local_runner.py
index 8f7979fd..f9ccfca0 100644
--- a/tradein-mvp/backend/scripts/domclick_local_runner.py
+++ b/tradein-mvp/backend/scripts/domclick_local_runner.py
@@ -755,6 +755,65 @@ def append_record(path: Path, rec: dict[str, Any]) -> None:
         fh.flush()
 
 
+# ── Enumerate cache (полный список id, чтобы НЕ гонять SERP заново) ────────────
+
+
+def write_enum_cache(path: str, offers: list[dict[str, Any]]) -> None:
+    """Перезаписать кэш-файл полным списком offer-id + sentinel завершённости.
+
+    Каждый offer пишется как {"id":..., "source_url":...} (только 2 поля),
+    финальной строкой — sentinel {"_enum_complete": true, "count": N}. Sentinel
+    означает «enumerate завершился полностью» — без него load_enum_cache считает
+    кэш неполным и заставляет re-enumerate.
+    """
+    cache_path = Path(path)
+    cache_path.parent.mkdir(parents=True, exist_ok=True)
+    with cache_path.open("w", encoding="utf-8") as fh:
+        for serp in offers:
+            row = {"id": serp.get("id"), "source_url": serp.get("source_url")}
+            fh.write(json.dumps(row, ensure_ascii=False) + "\n")
+        fh.write(json.dumps({"_enum_complete": True, "count": len(offers)}) + "\n")
+        fh.flush()
+    logger.info("enum-cache: wrote %d ids → %s", len(offers), path)
+
+
+def load_enum_cache(path: str) -> list[dict[str, Any]] | None:
+    """Загрузить полный список offer-id из кэша. None если файла нет/кэш неполный.
+
+    Кэш валиден ТОЛЬКО если последняя валидная строка — sentinel с
+    ``_enum_complete == True`` (enumerate завершился). Битые строки пропускаются;
+    нет sentinel / пустой / повреждённый → None (заставляет re-enumerate).
+    """
+    cache_path = Path(path)
+    if not cache_path.exists():
+        return None
+    records: list[dict[str, Any]] = []
+    with cache_path.open("r", encoding="utf-8") as fh:
+        for line in fh:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            if isinstance(rec, dict):
+                records.append(rec)
+    if not records:
+        return None
+    last = records[-1]
+    if not (isinstance(last, dict) and last.get("_enum_complete") is True):
+        return None  # кэш неполный (enumerate не завершился) → re-enumerate
+    offers: list[dict[str, Any]] = []
+    for rec in records[:-1]:
+        sid = rec.get("id")
+        if sid is None:
+            continue
+        offers.append({"id": str(sid), "source_url": rec.get("source_url")})
+    logger.info("enum-cache: loaded %d ids (complete)", len(offers))
+    return offers
+
+
 # ── SERP transport (playwright page navigation, чистый браузерный транспорт) ──
 
 
@@ -1220,9 +1279,22 @@ async def run(args: argparse.Namespace) -> None:
             logger.warning("warmup navigation failed (%s) — continuing", exc)
 
         # 1. Enumerate через фронтовый SERP (тот же чистый браузерный транспорт).
-        cards = await enumerate_cards(
-            page, done, args.limit_pages, target_new, enum_render, enum_jitter
-        )
+        # enum-cache: полный список id живёт в файле → следующий запуск ПРОПУСКАЕТ
+        # enumerate (~1.5-2ч) и сразу идёт в detail. detail-resume по `done`
+        # композится поверх: кэш даёт ВСЕ id, `done` (из JSONL) скипает готовые.
+        cards: list[dict[str, Any]] | None = None
+        if args.enum_cache:
+            cards = load_enum_cache(args.enum_cache)
+        if cards is None:
+            cards = await enumerate_cards(
+                page, done, args.limit_pages, target_new, enum_render, enum_jitter
+            )
+            # Sentinel пишем ТОЛЬКО после реально завершённого enumerate (offers
+            # полный) — прерванный/упавший свод кэш не перезаписывает.
+            if args.enum_cache:
+                write_enum_cache(args.enum_cache, cards)
+        else:
+            logger.info("enum-cache HIT: %d ids — enumerate ПРОПУЩЕН", len(cards))
         summary.enumerated = len(cards)
 
         if args.enumerate_only:
@@ -1312,6 +1384,12 @@ def build_arg_parser() -> argparse.ArgumentParser:
     )
     p.add_argument("--limit-pages", type=int, default=100, help="макс. страниц на room-бакет")
     p.add_argument("--enumerate-only", action="store_true", help="только перечислить, без detail")
+    p.add_argument(
+        "--enum-cache",
+        type=str,
+        default=None,
+        help="файл-кэш полного списка id (enumerate один раз → resume detail поверх)",
+    )
     p.add_argument(
         "--with-sold-similar",
         action="store_true",
-- 
2.45.3