From f65c33413c05a1fc3a59fca4de715cd2456552cc Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sun, 31 May 2026 11:24:12 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein):=20avito=20house-link=20=D0=B8?= =?UTF-8?q?=D0=B7=20detail=20SSR=20=E2=80=94=20SERP-=D0=BA=D0=B0=D1=80?= =?UTF-8?q?=D1=82=D0=BE=D1=87=D0=BA=D0=B0=20JS-only=20(#871)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit SERP-карточка отдаёт house-link («узнать больше о доме») только после client-side JS → в SSR-HTML, который тянет curl_cffi, его нет (verified live: SERP SSR 0/41 vs rendered DOM 21/60) → avito house-enrich мёртв (unique_houses=0 всегда). Detail-страница в SSR house-link содержит, а её мы и так фетчим для top-N. - avito_detail.py: house_catalog_url extraction fallback с nd-jk-details-button (новостройки) на любой a[href*='/catalog/houses/'] (вторичка) + strip ?context. - scrape_pipeline.py: Step 5b — энричим дома, найденные через detail-страницы (дедуп против SERP-derived Step 4). - test: secondary-маркер без nd-jk + query strip. Closes #871 --- .../backend/app/services/scrape_pipeline.py | 36 +++++++++++++++++++ .../app/services/scrapers/avito_detail.py | 11 ++++-- .../backend/tests/test_avito_detail_parse.py | 20 +++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index 8adcb4dc..b0c58f31 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -268,6 +268,7 @@ async def run_avito_pipeline( ) consecutive_blocks = 0 + detail_house_paths: set[str] = set() for idx, row in enumerate(priority_rows): source_url: str = row["source_url"] counters.detail_attempted += 1 @@ -278,6 +279,12 @@ async def run_avito_pipeline( enrichment_detail = await fetch_detail(item_url, cffi_session=session) if save_detail_enrichment(db, enrichment_detail): counters.detail_enriched += 1 + # #871: house-link есть в detail SSR (в SERP-карточке — JS-only). + # Копим catalog-пути для house-enrich ниже (Step 5b). + if enrichment_detail.house_catalog_url: + hp = urlparse(enrichment_detail.house_catalog_url).path + if hp: + detail_house_paths.add(hp) consecutive_blocks = 0 except (AvitoBlockedError, AvitoRateLimitedError) as e: consecutive_blocks += 1 @@ -310,6 +317,35 @@ async def run_avito_pipeline( jitter = random.uniform(0.8, 1.2) await asyncio.sleep(detail_delay * jitter) + # ── Step 5b: enrich houses, найденные через detail-страницы (#871) ── + # SERP-карточки больше не несут house-link (JS-render) → дома берём из + # detail SSR. Дедуп против обработанных в Step 4 (unique_house_paths). + new_house_paths = detail_house_paths - unique_house_paths + if enrich_houses and new_house_paths: + counters.unique_houses += len(new_house_paths) + nh_list = list(new_house_paths) + for h_idx, house_path in enumerate(nh_list): + try: + enrichment = await fetch_house_catalog(house_path, cffi_session=session) + house_counters = save_house_catalog_enrichment(db, enrichment) + counters.houses_enriched += 1 + hid = house_counters.get("house_id") + if hid: + touched_house_ids.add(int(hid)) + except (AvitoBlockedError, AvitoRateLimitedError): + logger.error( + "pipeline:house(detail) BLOCKED at %s — propagating", house_path + ) + raise + except Exception as e: + logger.warning( + "pipeline:house(detail) enrich failed for %s: %s", house_path, e + ) + counters.houses_failed += 1 + counters.errors.append(f"house(detail) {house_path}: {e}") + if h_idx < len(nh_list) - 1: + await asyncio.sleep(detail_delay) + logger.info( "pipeline:done anchor=(%.4f,%.4f) lots=%d (ins=%d/upd=%d) " "houses=%d/%d detail=%d/%d touched_house_ids=%d errors=%d", diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py index 4f9319d8..55a592c4 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py @@ -25,7 +25,7 @@ import re from dataclasses import dataclass, field from datetime import date from typing import Any -from urllib.parse import urljoin +from urllib.parse import urljoin, urlparse from curl_cffi.requests import AsyncSession from selectolax.parser import HTMLParser, Node @@ -319,12 +319,19 @@ def parse_detail_html(html: str, source_url: str) -> DetailEnrichment: metro_stations = _extract_metro(description) # ── House catalog link ─────────────────────────────────────────────────── + # Новостройки отдают кнопку ЖК (nd-jk-details-button); вторичка — ссылку + # «узнать больше о доме» под другим маркером. Fallback на любой /catalog/houses/ + # якорь: verified 2026-05-31 (#871) — эта ссылка есть в SSR detail-страницы, + # даже когда отсутствует в JS-рендеренной SERP-карточке (откуда мы брали 0). house_catalog_url: str | None = None house_link = tree.css_first("[data-marker='nd-jk-details-button']") + if house_link is None: + house_link = tree.css_first("a[href*='/catalog/houses/']") if house_link is not None: href = house_link.attributes.get("href", "") if href: - house_catalog_url = urljoin(AVITO_BASE, href) + # отбрасываем ?context-трекинг → чистый catalog-путь + house_catalog_url = urljoin(AVITO_BASE, urlparse(href).path) # ── Photo gallery ──────────────────────────────────────────────────────── photo_urls: list[str] = [] diff --git a/tradein-mvp/backend/tests/test_avito_detail_parse.py b/tradein-mvp/backend/tests/test_avito_detail_parse.py index a2808e39..14abe8f3 100644 --- a/tradein-mvp/backend/tests/test_avito_detail_parse.py +++ b/tradein-mvp/backend/tests/test_avito_detail_parse.py @@ -4,6 +4,8 @@ без сетевых запросов. """ +# ruff: noqa: E501 # HTML-фикстуры содержат длинные строки разметки + import pytest from app.services.scrapers.avito_detail import DetailEnrichment, parse_detail_html @@ -136,6 +138,24 @@ def test_house_catalog_url_full() -> None: assert result.house_catalog_url.startswith("https://www.avito.ru") +def test_house_catalog_url_secondary_marker_no_nd_jk() -> None: + """#871: вторичка без nd-jk-details-button → fallback на /catalog/houses/ якорь; + ?context-трекинг отброшен (verified live: house-link есть в detail SSR).""" + html = """ + +
№ 7683141312
+ 4 000 000 ₽ + + Узнать больше о доме + + """ + result = parse_detail_html(html, "https://www.avito.ru/test_7683141312") + assert ( + result.house_catalog_url + == "https://www.avito.ru/catalog/houses/ekaterinburg/ul_gotvalda_6k1/142792" + ) + + def test_bathroom_type_combined() -> None: html = """ -- 2.45.3