diff --git a/tradein-mvp/backend/scripts/domclick_local_runner.py b/tradein-mvp/backend/scripts/domclick_local_runner.py index 742a285f..de9542f8 100644 --- a/tradein-mvp/backend/scripts/domclick_local_runner.py +++ b/tradein-mvp/backend/scripts/domclick_local_runner.py @@ -14,28 +14,36 @@ DomClick фронтит QRATOR anti-bot. Проверено вживую (2026-0 САМОДОСТАТОЧНОСТЬ ================= -Никаких ``app.*`` импортов: вся логика Layer A (BFF enumerate) и Layer B +Никаких ``app.*`` импортов: вся логика Layer A (enumerate) и Layer B (detail ``__SSR_STATE__`` parse) скопирована ИНЛАЙН из ``app/services/scrapers/domclick.py`` + ``domclick_detail.py`` + ``repair_state_normalizer.py``. Зависимости: stdlib + ``playwright.async_api``. ТРАНСПОРТ ========= -Playwright Chromium, БЕЗ прокси (домашний IP): - • enumerate (BFF JSON) — ``context.request.get(url)`` (тот же сетевой - стек/IP/куки браузера) → парсим первый ``{`` … последний ``}``; +Playwright Chromium, БЕЗ прокси (домашний IP). И enumerate, и detail идут +ОДНИМ браузерным транспортом ``page.goto`` (verified 2026-06-27): + • enumerate — ``page.goto(SERP_url)`` на фронтовый + ``ekaterinburg.domclick.ru/search?...`` (SSR). Прямой BFF + ``bff-search-web.domclick.ru/api/offers/v1`` режет QRATOR-403 уже с + домашнего IP, а фронтовый SERP грузится чисто (тот же транспорт, что + card-detail). id офферов берём из DOM (``a[href*="/card/"]``), total — + из ``
-wrap).
-
- Raises:
- DomClickBlockedError: QRATOR/captcha маркеры в теле.
- ValueError: JSON не найден / не dict.
- """
- if _is_block_text(body):
- raise DomClickBlockedError("BFF: QRATOR block page detected")
- start = body.find("{")
- end = body.rfind("}")
- if start == -1 or end == -1 or end <= start:
- raise ValueError(f"No JSON object in BFF response (len={len(body)})")
- data = json.loads(body[start : end + 1])
- if not isinstance(data, dict):
- raise ValueError(f"BFF JSON is not a dict: {type(data)}")
- return data
-
-
-def _build_offers_url(rooms: str, gte: int | None, lte: int | None, offset: int) -> str:
+def _build_serp_url(rooms: str, gte: int | None, lte: int | None, offset: int) -> str:
+ """URL фронтового SERP. urlencode кодирует "5+" → "5%2B" (иначе фронт читает
+ '+' как пробел и игнорит room-фильтр)."""
params: list[tuple[str, str]] = [
- ("address", _EKB_ADDRESS_GUID),
- ("aids", _EKB_AREA_ID),
("deal_type", "sale"),
("category", "living"),
("offer_type", "flat"),
- ("rooms", rooms),
- ("sort", "qi"),
- ("sort_dir", "desc"),
+ ("aids", _EKB_AREA_ID),
+ ("from", "topline2020"),
("offset", str(offset)),
- ("limit", str(PAGE_SIZE)),
- ]
- if gte is not None:
- params.append(("sale_price__gte", str(gte)))
- if lte is not None:
- params.append(("sale_price__lte", str(lte)))
- return f"{_BFF_BASE}/api/offers/v1?{urlencode(params)}"
-
-
-def _build_count_url(rooms: str, gte: int | None, lte: int | None) -> str:
- params: list[tuple[str, str]] = [
- ("address", _EKB_ADDRESS_GUID),
- ("aids", _EKB_AREA_ID),
- ("deal_type", "sale"),
- ("category", "living"),
- ("offer_type", "flat"),
("rooms", rooms),
- ("sort", "qi"),
- ("sort_dir", "desc"),
]
if gte is not None:
params.append(("sale_price__gte", str(gte)))
if lte is not None:
params.append(("sale_price__lte", str(lte)))
- return f"{_BFF_BASE}/api/offers/count/v1?{urlencode(params)}"
+ return f"{_SERP_HOST}/search?{urlencode(params)}"
-def _is_geo_ok(item: dict[str, Any]) -> bool:
- """bbox-guard ЕКБ. lat/lon обязательны и в пределах bbox."""
- loc = item.get("location") or {}
- lat = loc.get("lat")
- lon = loc.get("lon")
- if lat is None or lon is None:
- return False
- try:
- latf, lonf = float(lat), float(lon)
- except (ValueError, TypeError):
- return False
- return _EKB_LAT_MIN <= latf <= _EKB_LAT_MAX and _EKB_LON_MIN <= lonf <= _EKB_LON_MAX
-
-
-# ── SERP item → плоский dict (инлайн из domclick.py _map_item) ────────────────
-
-
-def map_serp_item(item: dict[str, Any], force_rooms: int | None) -> dict[str, Any] | None:
- """offer-item из BFF → SERP-dict. None при price<=0 / без id / ошибке."""
- try:
- price = item.get("price") or 0
- if price <= 0:
- return None
- item_id = item.get("id")
- if item_id is None:
- return None
- sid = str(item_id)
-
- path = item.get("path") or ""
- if path.startswith("/"):
- path = _CARD_HOST_FALLBACK + path
-
- loc = item.get("location") or {}
- oi = item.get("objectInfo") or {}
- house = item.get("house") or {}
- addr = item.get("address") or {}
-
- area_raw = oi.get("area")
- floor_raw = oi.get("floor")
- floors_raw = house.get("floors")
- year_raw = house.get("buildYear")
- sq_raw = item.get("squarePrice")
-
- if force_rooms is not None:
- rooms: int | None = force_rooms
- else:
- rr = oi.get("rooms")
- rooms = int(rr) if rr is not None else None
-
- return {
- "id": sid,
- "source_url": path,
- "price_rub": int(price),
- "square_price": int(sq_raw) if sq_raw else None,
- "area_m2": float(area_raw) if area_raw else None,
- "rooms": rooms,
- "floor": int(floor_raw) if floor_raw else None,
- "total_floors": int(floors_raw) if floors_raw else None,
- "year_built": int(year_raw) if year_raw else None,
- "lat": loc.get("lat"),
- "lon": loc.get("lon"),
- "address": addr.get("displayName"),
- "is_apartment": oi.get("isApartment"),
- "offer_region_name": item.get("offerRegionName"),
- "description": item.get("description"),
- "is_sber_collateral": item.get("isSberCollateral"),
- "published_date": item.get("publishedDate"),
- }
- except Exception:
- logger.warning("map_serp_item failed id=%r", item.get("id"), exc_info=True)
+def _parse_serp_total(title: str) -> int | None:
+ """total из ("… 2 269 объявлений"). None если числа нет."""
+ if not title:
return None
+ m = _SERP_TOTAL_RE.search(title)
+ if not m:
+ return None
+ digits = re.sub(r"\D", "", m.group(1))
+ return int(digits) if digits else None
+
+
+def _canon_card_url(sid: str, url: str | None) -> str:
+ """Канонизируем source_url к /card/sale__flat__, если href неполный."""
+ if url and url.startswith("http") and "/card/" in url:
+ return url
+ return f"{_SERP_HOST}/card/sale__flat__{sid}"
+
+
+def _serp_offer_to_dict(raw: Any) -> dict[str, Any] | None:
+ """offer из _SERP_IDS_JS ({id,url}) → минимальный SERP-dict {id, source_url}.
+ Layer-A (price/area/...) теперь добирается из card-SSR в parse_detail_html."""
+ if not isinstance(raw, dict):
+ return None
+ sid = raw.get("id")
+ if sid is None:
+ return None
+ sid = str(sid)
+ if not sid.isdigit():
+ return None
+ return {"id": sid, "source_url": _canon_card_url(sid, raw.get("url"))}
# ── SSR state extraction (инлайн из domclick_detail.py) ──────────────────────
@@ -601,6 +541,27 @@ def parse_detail_html(html: str, source_url: str) -> dict[str, Any]:
views = _to_int(pc.get("viewsCount"))
calls = _to_int(pc.get("callsCount"))
+ # Layer-A из card-SSR (раньше шёл из BFF serp; теперь enumerate даёт
+ # только id+url). Пути verified 2026-06-27: цена/squarePrice — в
+ # priceInfo; area/rooms/floor — в objectInfo; total_floors — houseInfo;
+ # lat/lon/address — productCard.address(.position).
+ price_info = pc.get("priceInfo") or {}
+ address = pc.get("address") or {}
+ position = address.get("position") or {}
+ price_rub = _to_int(price_info.get("price"))
+ square_price = _to_int(price_info.get("squarePrice"))
+ area_m2 = _pos_float(oi.get("area"))
+ rooms = _to_int(oi.get("rooms"))
+ floor = _to_int(oi.get("floor"))
+ total_floors = _to_int(hi.get("floors")) or _to_int(
+ (pc.get("house") or {}).get("info", {}).get("floors")
+ )
+ lat = _to_float(position.get("lat"))
+ lon = _to_float(position.get("lon"))
+ address_name = address.get("displayName")
+ is_apartment = oi.get("isApartment")
+ description = oi.get("description")
+
# priceHistory → нормализованные записи (skip при битой дате/без цены).
raw_history = (pc.get("priceInfo") or {}).get("priceHistory") or []
price_changes: list[dict[str, Any]] = []
@@ -664,6 +625,19 @@ def parse_detail_html(html: str, source_url: str) -> dict[str, Any]:
)
return {
+ # Layer-A (из card-SSR; раньше из BFF serp).
+ "price_rub": price_rub,
+ "square_price": square_price,
+ "area_m2": area_m2,
+ "rooms": rooms,
+ "floor": floor,
+ "total_floors": total_floors,
+ "lat": lat,
+ "lon": lon,
+ "address": address_name,
+ "is_apartment": is_apartment,
+ "description": description,
+ # Layer-B.
"repair_state": repair_state,
"repair_raw": repair_raw,
"living_area_m2": living,
@@ -698,27 +672,29 @@ def build_record(
d = detail or {}
repair_state = d.get("repair_state")
if not repair_state:
- # fallback-инференс из SERP-описания (как в проде для пустого структурного)
- repair_state = infer_repair_state_from_text(serp.get("description"))
+ # fallback-инференс из описания (теперь из card-SSR; serp-описания больше нет)
+ repair_state = infer_repair_state_from_text(d.get("description") or serp.get("description"))
is_sber = serp.get("is_sber_collateral")
if not is_sber and d.get("collateral_sber"):
is_sber = True
+ # Layer-A: приоритет card-SSR (d), фолбэк на serp (на случай если serp когда-то
+ # снова начнёт отдавать поля). enumerate теперь даёт только id+source_url.
return {
"id": serp.get("id"),
"source_url": serp.get("source_url"),
"status": status,
"fetched_at": datetime.now(UTC).isoformat(),
- "price_rub": serp.get("price_rub"),
- "area_m2": serp.get("area_m2"),
- "rooms": serp.get("rooms"),
- "floor": serp.get("floor"),
- "total_floors": serp.get("total_floors"),
+ "price_rub": d.get("price_rub") or serp.get("price_rub"),
+ "area_m2": d.get("area_m2") or serp.get("area_m2"),
+ "rooms": d.get("rooms") if d.get("rooms") is not None else serp.get("rooms"),
+ "floor": d.get("floor") or serp.get("floor"),
+ "total_floors": d.get("total_floors") or serp.get("total_floors"),
"year_built": d.get("year_built") or serp.get("year_built"),
- "lat": serp.get("lat"),
- "lon": serp.get("lon"),
- "address": serp.get("address"),
+ "lat": d.get("lat") or serp.get("lat"),
+ "lon": d.get("lon") or serp.get("lon"),
+ "address": d.get("address") or serp.get("address"),
"repair_state": repair_state,
"repair_raw": d.get("repair_raw"),
"living_area_m2": d.get("living_area_m2"),
@@ -734,11 +710,16 @@ def build_record(
"is_sber_collateral": is_sber,
"avm": d.get("avm") or {},
"raw_extra": {
- "square_price": serp.get("square_price"),
- "is_apartment": serp.get("is_apartment"),
+ "square_price": d.get("square_price") or serp.get("square_price"),
+ "is_apartment": (
+ d.get("is_apartment")
+ if d.get("is_apartment") is not None
+ else serp.get("is_apartment")
+ ),
"offer_region_name": serp.get("offer_region_name"),
"published_date": serp.get("published_date"),
**(d.get("raw_extra") or {}),
+ **({"sold_similar": d["sold_similar"]} if d.get("sold_similar") else {}),
},
}
@@ -774,79 +755,103 @@ def append_record(path: Path, rec: dict[str, Any]) -> None:
fh.flush()
-# ── BFF transport (playwright APIRequestContext) ─────────────────────────────
+# ── SERP transport (playwright page navigation, чистый браузерный транспорт) ──
-async def fetch_bff_json(context: Any, url: str) -> dict[str, Any]:
- """GET BFF через context.request (тот же IP/куки браузера) → dict."""
- resp = await context.request.get(
- url,
- headers={
- "Accept": "application/json, text/plain, */*",
- "Referer": "https://ekaterinburg.domclick.ru/search?deal_type=sale",
- "Accept-Language": "ru-RU,ru;q=0.9",
- },
- timeout=60000,
- )
- body = await resp.text()
- if resp.status == 403:
- raise DomClickBlockedError(f"BFF 403 for {url}")
- return _extract_json(body)
+async def fetch_serp_page(
+ page: Any,
+ url: str,
+ render_wait: tuple[float, float],
+) -> tuple[list[dict[str, Any]], int | None, int]:
+ """Навигация на фронтовый SERP → (offers, total, http_status).
+
+ offers — список {id, source_url} из DOM; total — из .
+ Raises:
+ DomClickBlockedError: 403 или QRATOR-маркеры в title.
+ """
+ resp = await page.goto(url, wait_until="domcontentloaded", timeout=70000)
+ await asyncio.sleep(random.uniform(*render_wait)) # дать SSR отрендериться
+ status = resp.status if resp else 0
+ title = ""
+ try:
+ title = (await page.title()) or ""
+ except Exception:
+ title = ""
+ if status == 403 or _is_block_text(title):
+ raise DomClickBlockedError(f"SERP block status={status} title={title!r}")
+ raw = await page.evaluate(_SERP_IDS_JS)
+ offers: list[dict[str, Any]] = []
+ if isinstance(raw, list):
+ for item in raw:
+ serp = _serp_offer_to_dict(item)
+ if serp is not None:
+ offers.append(serp)
+ total = _parse_serp_total(title)
+ return offers, total, status
-async def get_count(context: Any, rooms: str, gte: int | None, lte: int | None) -> int:
- data = await fetch_bff_json(context, _build_count_url(rooms, gte, lte))
- res = data.get("result") or {}
- raw = res.get("snippetsCount")
- return int(raw) if raw else 0
+# ── Enumerate (mirror domclick.py _sweep_bucket / _paginate, на title-total) ──
-# ── Enumerate (mirror domclick.py _sweep_bucket / _paginate) ─────────────────
+def _add_offers(
+ offers: list[dict[str, Any]],
+ seen: set[str],
+ done: set[str],
+ out: list[dict[str, Any]],
+ target_new: int | None,
+) -> int:
+ """Добавить новые офферы в out (dedup по seen/done). → число добавленных.
+ Бросает _EnumLimitReachedError при достижении target_new."""
+ added = 0
+ for serp in offers:
+ sid = serp["id"]
+ if sid in seen or sid in done:
+ continue
+ seen.add(sid)
+ out.append(serp)
+ added += 1
+ if target_new is not None and len(out) >= target_new:
+ raise _EnumLimitReachedError
+ return added
async def _paginate(
- context: Any,
+ page: Any,
rooms: str,
gte: int | None,
lte: int | None,
+ total: int | None,
seen: set[str],
done: set[str],
out: list[dict[str, Any]],
limit_pages: int,
target_new: int | None,
+ render_wait: tuple[float, float],
enum_jitter: tuple[float, float],
+ start_page: int,
) -> None:
- force_rooms = 0 if rooms == "st" else None
- page_idx = 0
+ """Пагинация SERP с offset=start_page*20 далее. Стоп: достигли total/cap,
+ кончились limit_pages, или страница не дала новых id."""
+ bound = OFFSET_CAP if total is None else min(total, OFFSET_CAP)
+ page_idx = start_page
while page_idx < limit_pages:
- offset = page_idx * PAGE_SIZE
- if offset >= OFFSET_CAP:
- break
- data = await fetch_bff_json(context, _build_offers_url(rooms, gte, lte, offset))
- items = (data.get("result") or {}).get("items") or []
- if not items:
- break
- for item in items:
- serp = map_serp_item(item, force_rooms)
- if serp is None:
- continue
- if not _is_geo_ok(item):
- continue
- sid = serp["id"]
- if sid in seen or sid in done:
- continue
- seen.add(sid)
- out.append(serp)
- if target_new is not None and len(out) >= target_new:
- raise _EnumLimitReachedError
- page_idx += 1
- if len(items) < PAGE_SIZE:
+ offset = page_idx * _SERP_PAGE_SIZE
+ if offset >= bound:
break
await asyncio.sleep(random.uniform(*enum_jitter))
+ offers, _, _ = await fetch_serp_page(
+ page, _build_serp_url(rooms, gte, lte, offset), render_wait
+ )
+ if not offers:
+ break
+ added = _add_offers(offers, seen, done, out, target_new)
+ page_idx += 1
+ if added == 0:
+ break
async def _sweep_bucket(
- context: Any,
+ page: Any,
rooms: str,
gte: int | None,
lte: int | None,
@@ -855,86 +860,120 @@ async def _sweep_bucket(
out: list[dict[str, Any]],
limit_pages: int,
target_new: int | None,
+ render_wait: tuple[float, float],
enum_jitter: tuple[float, float],
depth: int,
) -> None:
- count = await get_count(context, rooms, gte, lte)
- if count == 0:
- return
- if count > OFFSET_CAP and depth < MAX_SPLIT_DEPTH:
- _gte = gte if gte is not None else 0
- _lte = lte if lte is not None else LTE_MAX
- span = _lte - _gte
- if span <= MIN_PRICE_SPAN:
- logger.warning(
- "rooms=%r price=[%s,%s] count=%d > cap but span<=%d — paginating as-is",
+ """offset=0 → total + page-0 офферы. total>cap → price-bisection, иначе
+ пагинируем остаток. total читаем из (чистый фронтовый транспорт)."""
+ offers0, total, _ = await fetch_serp_page(
+ page, _build_serp_url(rooms, gte, lte, 0), render_wait
+ )
+ _add_offers(offers0, seen, done, out, target_new)
+ if not total:
+ # нет числа / 0 объявлений: офферов на page-0 нет → выходим; есть → допагинируем.
+ if offers0:
+ await _paginate(
+ page,
rooms,
gte,
lte,
- count,
- MIN_PRICE_SPAN,
+ None,
+ seen,
+ done,
+ out,
+ limit_pages,
+ target_new,
+ render_wait,
+ enum_jitter,
+ 1,
)
- await _paginate(
- context, rooms, gte, lte, seen, done, out, limit_pages, target_new, enum_jitter
+ return
+ if total > OFFSET_CAP and depth < MAX_SPLIT_DEPTH:
+ _gte = gte if gte is not None else 0
+ _lte = lte if lte is not None else LTE_MAX
+ span = _lte - _gte
+ if span > MIN_PRICE_SPAN:
+ mid = (_gte + _lte) // 2
+ logger.info(
+ "rooms=%r total=%d > cap — split [%d,%d]+[%d,%d]",
+ rooms,
+ total,
+ _gte,
+ mid,
+ mid + 1,
+ _lte,
+ )
+ await _sweep_bucket(
+ page,
+ rooms,
+ _gte,
+ mid,
+ seen,
+ done,
+ out,
+ limit_pages,
+ target_new,
+ render_wait,
+ enum_jitter,
+ depth + 1,
+ )
+ await _sweep_bucket(
+ page,
+ rooms,
+ mid + 1,
+ _lte,
+ seen,
+ done,
+ out,
+ limit_pages,
+ target_new,
+ render_wait,
+ enum_jitter,
+ depth + 1,
)
return
- mid = (_gte + _lte) // 2
- logger.info(
- "rooms=%r count=%d > cap — split [%d,%d]+[%d,%d]",
+ logger.warning(
+ "rooms=%r price=[%s,%s] total=%d > cap но span<=%d — пагинируем как есть",
rooms,
- count,
- _gte,
- mid,
- mid + 1,
- _lte,
- )
- await _sweep_bucket(
- context,
- rooms,
- _gte,
- mid,
- seen,
- done,
- out,
- limit_pages,
- target_new,
- enum_jitter,
- depth + 1,
- )
- await _sweep_bucket(
- context,
- rooms,
- mid + 1,
- _lte,
- seen,
- done,
- out,
- limit_pages,
- target_new,
- enum_jitter,
- depth + 1,
- )
- else:
- await _paginate(
- context, rooms, gte, lte, seen, done, out, limit_pages, target_new, enum_jitter
+ gte,
+ lte,
+ total,
+ MIN_PRICE_SPAN,
)
+ await _paginate(
+ page,
+ rooms,
+ gte,
+ lte,
+ total,
+ seen,
+ done,
+ out,
+ limit_pages,
+ target_new,
+ render_wait,
+ enum_jitter,
+ 1,
+ )
async def enumerate_cards(
- context: Any,
+ page: Any,
done: set[str],
limit_pages: int,
target_new: int | None,
+ render_wait: tuple[float, float],
enum_jitter: tuple[float, float],
) -> list[dict[str, Any]]:
- """Свод ЕКБ-вторички через BFF: все room-бакеты, dedup, skip done."""
+ """Свод ЕКБ-вторички через фронтовый SERP: все room-бакеты, dedup, skip done."""
out: list[dict[str, Any]] = []
seen: set[str] = set()
try:
for bucket in ROOM_BUCKETS:
logger.info("enumerate: bucket rooms=%r (collected=%d)", bucket, len(out))
await _sweep_bucket(
- context,
+ page,
bucket,
None,
None,
@@ -943,6 +982,7 @@ async def enumerate_cards(
out,
limit_pages,
target_new,
+ render_wait,
enum_jitter,
0,
)
@@ -975,10 +1015,90 @@ async def fetch_card_html(
return html, title, status
+# JS: дозабор offer-card v3 (AVM price_prediction + опц. sold_similar) из
+# контекста УЖЕ загруженной страницы — та же сессия/куки, что прошли QRATOR.
+_OFFER_CARD_EXTRAS_JS = """async (args) => {
+ const [id, withSold, soldLimit] = args;
+ const out = {};
+ const get = async (u) => {
+ try {
+ const r = await fetch(u, {credentials: 'include'});
+ return r.ok ? await r.json() : {__status: r.status};
+ } catch (e) {
+ return {__err: String(e)};
+ }
+ };
+ out.price_prediction = await get(
+ 'https://offer-card.domclick.ru/api/v3/offers/price_prediction?offer_id=' + id
+ );
+ if (withSold) {
+ out.sold_similar = await get(
+ 'https://offer-card.domclick.ru/api/v3/offers/sold_similar/flat/' + id +
+ '?in_house=false&limit=' + soldLimit + '&offset=0'
+ );
+ }
+ return out;
+}"""
+
+
+def _parse_offer_card_avm(price_prediction: Any) -> dict[str, Any] | None:
+ """price_prediction.result → AVM-форма (как d["avm"] из SSR). None если пусто."""
+ if not isinstance(price_prediction, dict):
+ return None
+ result = price_prediction.get("result")
+ if not isinstance(result, dict):
+ return None
+ market_price = result.get("market_price")
+ if market_price is None:
+ return None
+ avm = {
+ "market_price": market_price,
+ "min": result.get("min_market_price"),
+ "max": result.get("max_market_price"),
+ "repair_quality": result.get("repair_quality"),
+ "rent_long": result.get("rent_long"),
+ "rent_short": result.get("rent_short"),
+ }
+ return {k: v for k, v in avm.items() if v is not None}
+
+
+def _parse_offer_card_sold(sold_similar: Any) -> dict[str, Any] | None:
+ """sold_similar → {total, items}. None если нет result.items."""
+ if not isinstance(sold_similar, dict):
+ return None
+ result = sold_similar.get("result")
+ if not isinstance(result, dict):
+ return None
+ items = result.get("items")
+ if not isinstance(items, list):
+ return None
+ pagination = sold_similar.get("pagination") or {}
+ total = pagination.get("total") if isinstance(pagination, dict) else None
+ return {"total": total, "items": items}
+
+
+async def fetch_offer_card_extras(
+ page: Any,
+ item_id: str,
+ with_sold_similar: bool,
+ sold_limit: int,
+) -> dict[str, Any]:
+ """Дозабор AVM (всегда) + sold_similar (опц.) с offer-card v3 из контекста
+ загруженной страницы. None-safe: любая ошибка/__status/__err → поле None."""
+ raw = await page.evaluate(_OFFER_CARD_EXTRAS_JS, [str(item_id), with_sold_similar, sold_limit])
+ if not isinstance(raw, dict):
+ return {"avm": None, "sold_similar": None}
+ avm = _parse_offer_card_avm(raw.get("price_prediction"))
+ sold = _parse_offer_card_sold(raw.get("sold_similar")) if with_sold_similar else None
+ return {"avm": avm, "sold_similar": sold}
+
+
async def process_card(
page: Any,
serp: dict[str, Any],
render_wait: tuple[float, float],
+ with_sold_similar: bool,
+ sold_limit: int,
) -> tuple[str, dict[str, Any] | None]:
"""Один detail-fetch с одним ретраем на блок. → (status, detail|None)."""
url = serp["source_url"]
@@ -988,6 +1108,16 @@ async def process_card(
if http_status == 403 or _is_block_text(title) or _is_block_text(html[:4096]):
raise DomClickBlockedError(f"403/challenge title={title!r}")
detail = parse_detail_html(html, url)
+ try:
+ extras = await fetch_offer_card_extras(
+ page, serp["id"], with_sold_similar, sold_limit
+ )
+ if extras.get("avm"):
+ detail["avm"] = extras["avm"]
+ if extras.get("sold_similar"):
+ detail["sold_similar"] = extras["sold_similar"]
+ except Exception as exc:
+ logger.warning("card %s offer-card extras failed: %s", serp["id"], exc)
return "ok", detail
except DomClickBlockedError as exc:
if attempt == 1:
@@ -1049,6 +1179,9 @@ async def run(args: argparse.Namespace) -> None:
summary = Summary()
render_wait = (args.render_min, args.render_max)
target_new = args.limit if args.limit and args.limit > 0 else None
+ # SERP-навигация требует render-wait на SSR; enum_jitter — доп. пейсинг между
+ # страницами (SERP-навигаций много при свипе, бережём home-IP).
+ enum_render = (max(2.0, args.render_min - 2.0), max(3.0, args.render_max - 2.0))
enum_jitter = (1.5, 4.0)
# ВАЖНО: launch(headless=True) = СТАРЫЙ headless → QRATOR 403 даже с дом. IP.
@@ -1069,7 +1202,7 @@ async def run(args: argparse.Namespace) -> None:
)
page = await context.new_page()
- # Warm-up: устанавливаем QRATOR-куки .domclick.ru перед BFF-запросами.
+ # Warm-up: устанавливаем QRATOR-куки .domclick.ru перед SERP-навигацией.
try:
await page.goto(_WARMUP_URL, wait_until="domcontentloaded", timeout=70000)
await asyncio.sleep(random.uniform(2.0, 4.0))
@@ -1077,8 +1210,10 @@ async def run(args: argparse.Namespace) -> None:
except Exception as exc:
logger.warning("warmup navigation failed (%s) — continuing", exc)
- # 1. Enumerate.
- cards = await enumerate_cards(context, done, args.limit_pages, target_new, enum_jitter)
+ # 1. Enumerate через фронтовый SERP (тот же чистый браузерный транспорт).
+ cards = await enumerate_cards(
+ page, done, args.limit_pages, target_new, enum_render, enum_jitter
+ )
summary.enumerated = len(cards)
if args.enumerate_only:
@@ -1102,7 +1237,9 @@ async def run(args: argparse.Namespace) -> None:
summary.skipped_done += 1
continue
- status, detail = await process_card(page, serp, render_wait)
+ status, detail = await process_card(
+ page, serp, render_wait, args.with_sold_similar, args.sold_limit
+ )
rec = build_record(serp, detail, status)
append_record(out_path, rec)
done.add(sid)
@@ -1166,6 +1303,14 @@ def build_arg_parser() -> argparse.ArgumentParser:
)
p.add_argument("--limit-pages", type=int, default=100, help="макс. страниц на room-бакет")
p.add_argument("--enumerate-only", action="store_true", help="только перечислить, без detail")
+ p.add_argument(
+ "--with-sold-similar",
+ action="store_true",
+ help="дозабор sold_similar с offer-card v3 (+1 запрос/карточку)",
+ )
+ p.add_argument(
+ "--sold-limit", type=int, default=5, help="limit для sold_similar (если включён)"
+ )
p.add_argument(
"--headed",
action="store_true",