From 464e9faf7bd63d22d814db5b452feb48243b2f67 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Fri, 19 Jun 2026 19:21:49 +0300 Subject: [PATCH] =?UTF-8?q?fix(scrapers):=20avito=5Fhouses=20=D1=87=D0=B8?= =?UTF-8?q?=D1=82=D0=B0=D0=B5=D1=82=20MFE=20props-=D0=B2=D0=BB=D0=BE=D0=B6?= =?UTF-8?q?=D0=B5=D0=BD=D0=BD=D0=BE=D1=81=D1=82=D1=8C=20=D0=B2=D0=B8=D0=B4?= =?UTF-8?q?=D0=B6=D0=B5=D1=82=D0=BE=D0=B2=20=E2=80=94=20=D0=BE=D1=82=D0=B7?= =?UTF-8?q?=D1=8B=D0=B2=D1=8B/=D0=B8=D1=81=D1=82=D0=BE=D1=80=D0=B8=D1=8F/?= =?UTF-8?q?=D1=82=D0=B8=D0=BF=20=D0=96=D0=9A=20=D1=82=D0=B5=D0=BF=D0=B5?= =?UTF-8?q?=D1=80=D1=8C=20=D0=B4=D0=BE=D0=BB=D0=B5=D1=82=D0=B0=D1=8E=D1=82?= =?UTF-8?q?=20(#1789)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/services/scrapers/avito_houses.py | 43 +- .../tests/fixtures/avito_house_572538.html | 8 + .../backend/tests/test_avito_houses_parse.py | 399 +++++++++++------- 3 files changed, 302 insertions(+), 148 deletions(-) create mode 100644 tradein-mvp/backend/tests/fixtures/avito_house_572538.html diff --git a/tradein-mvp/backend/app/services/scrapers/avito_houses.py b/tradein-mvp/backend/app/services/scrapers/avito_houses.py index 1a433abc..787ddcfb 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_houses.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_houses.py @@ -4,6 +4,11 @@ и разбирает 10 виджетов: housePage, reviews, miniSerp, housePlacementHistory, recommendations. +Avito (≈2026, MFE) перенёс полезную нагрузку виджетов на уровень глубже — +под ключ с именем самого виджета (props['reviews']['entries'], +props['miniSerp']['items'], props['housePlacementHistory']['items'] и т.д.). +Разворачивание делает _widget_payload с fallback на старую форму (props напрямую). + Flow: fetch_house_catalog(house_url) → HTTP GET (curl_cffi chrome120) @@ -316,6 +321,26 @@ def _get_widget(placeholders: list[dict[str, Any]], widget_type: str) -> dict[st return None +def _widget_payload(widget: dict[str, Any], widget_type: str) -> dict[str, Any]: + """Возвращает полезную нагрузку виджета, разворачивая type-вложенность. + + Avito (≈2026, MFE) переместил данные виджета на уровень глубже — + под ключ с именем самого виджета: + новый: widget['props'][widget_type] = {... entries / items ...} + старый: widget['props'] = {... entries / items ...} + + Хелпер пробует новую форму (props[widget_type]), fallback на старую + (props напрямую) → поддержка обеих форм. + """ + props = widget.get("props", {}) + if not isinstance(props, dict): + return {} + nested = props.get(widget_type) + if isinstance(nested, dict): + return nested + return props + + # --------------------------------------------------------------------------- # Widget parsers # --------------------------------------------------------------------------- @@ -427,7 +452,8 @@ def _parse_reviews(widget: dict[str, Any]) -> tuple[list[HouseReview], str | Non reviews: list[HouseReview] = [] next_page_url: str | None = None - entries = widget.get("props", {}).get("entries", []) + payload = _widget_payload(widget, "reviews") + entries = payload.get("entries", []) for entry in entries: entry_type = entry.get("type") value = entry.get("value", {}) @@ -471,7 +497,7 @@ def _parse_mini_serp(widget: dict[str, Any]) -> list[MiniSerpListing]: """Парсит виджет miniSerp → список активных объявлений в доме.""" listings: list[MiniSerpListing] = [] - items = widget.get("props", {}).get("items", []) + items = _widget_payload(widget, "miniSerp").get("items", []) for item in items: geo = item.get("geo", {}) colors: list[str] = geo.get("colors", []) @@ -506,7 +532,7 @@ def _parse_placement_history(widget: dict[str, Any]) -> list[PlacementHistoryIte """ result: list[PlacementHistoryItem] = [] - items = widget.get("props", {}).get("items", []) + items = _widget_payload(widget, "housePlacementHistory").get("items", []) for item in items: item_copy = {k: v for k, v in item.items() if k != "itemImage"} result.append( @@ -530,7 +556,7 @@ def _parse_recommendations(widget: dict[str, Any]) -> list[RecommendationItem]: """Парсит виджет recommendations → lightweight список похожих объявлений.""" result: list[RecommendationItem] = [] - items = widget.get("props", {}).get("items", []) + items = _widget_payload(widget, "recommendations").get("items", []) for item in items: additional_info: list[str] = item.get("additionalInfo", []) address = additional_info[0] if additional_info else None @@ -636,6 +662,8 @@ def parse_houses_state(state: dict[str, Any], house_url: str) -> HouseCatalogEnr reviews_widget = _get_widget(placeholders, "reviews") if reviews_widget is not None: reviews, next_page_url = _parse_reviews(reviews_widget) + if not reviews: + logger.warning("Виджет 'reviews' найден, но 0 отзывов распарсено — проверь шейп props") else: logger.debug("Виджет 'reviews' не найден — пропускаем") @@ -652,6 +680,10 @@ def parse_houses_state(state: dict[str, Any], house_url: str) -> HouseCatalogEnr history_widget = _get_widget(placeholders, "housePlacementHistory") if history_widget is not None: placement_history = _parse_placement_history(history_widget) + if not placement_history: + logger.warning( + "Виджет 'housePlacementHistory' найден, но 0 items распарсено — проверь шейп props" + ) else: logger.debug("Виджет 'housePlacementHistory' не найден — пропускаем") @@ -842,7 +874,8 @@ def _persist_house(db: Session, h: HouseInfo, house_url: str) -> int: lat = COALESCE(houses.lat, CAST(:lat AS double precision)), lon = COALESCE(houses.lon, CAST(:lon AS double precision)), year_built = COALESCE(houses.year_built, CAST(:year_built AS int)), - last_scraped_at = NOW() + last_scraped_at = NOW(), + avito_validated_at = NOW() WHERE id = CAST(:house_id AS bigint) """), { diff --git a/tradein-mvp/backend/tests/fixtures/avito_house_572538.html b/tradein-mvp/backend/tests/fixtures/avito_house_572538.html new file mode 100644 index 00000000..9643214d --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/avito_house_572538.html @@ -0,0 +1,8 @@ + + + + diff --git a/tradein-mvp/backend/tests/test_avito_houses_parse.py b/tradein-mvp/backend/tests/test_avito_houses_parse.py index d604bd8d..e40a13f2 100644 --- a/tradein-mvp/backend/tests/test_avito_houses_parse.py +++ b/tradein-mvp/backend/tests/test_avito_houses_parse.py @@ -30,112 +30,143 @@ FIXTURES_DIR = Path(__file__).parent / "fixtures" # --------------------------------------------------------------------------- MINIMAL_STATE: dict = { - "data": {"data": {"page": {"placeholders": [ - {"type": "breadcrumbs"}, - {"type": "title"}, - {"type": "gallery", "props": {}}, - { - "type": "housePage", - "props": {"developmentData": { - "avitoId": 3171365, - "id": "MTE3LjM3MDg5MA", - "title": "Akademika Postovskogo 17a", - "address": "ул. Постовского, 17а", - "fullAddress": "Свердловская обл., Екатеринбург, ул. Постовского, 17а", - "coords": {"lat": 56.790699, "lng": 60.580191}, - "aboutDevelopment": {"expandParams": {"items": [ - {"title": "Параметры дома", "params": [ - {"type": "Год постройки", "value": "2019"}, - {"type": "Этажей", "value": "25"}, - {"type": "Тип дома", "value": "Монолитный"}, - {"type": "Пассажирский лифт", "value": "2"}, - {"type": "Консьерж", "value": "Да"}, - {"type": "Закрытый двор", "value": "Да"}, - {"type": "Детская площадка", "value": "Да"}, - ]}, - ]}}, - "developer": {"name": "Атомстройкомплекс", "key": "atomstroy"}, - "mapPreview": { - "distance": "в 5 минутах ходьбы", - "objects": "школа, парк, аптека", - "pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}], - }, - "ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}}, - "ratingSummaryStat": { - "ratingStat": [{"score": 5, "count": 4}, {"score": 4, "count": 2}], - "reviewCount": 6, - }, - }}, - }, - { - "type": "reviews", - "props": {"entries": [ - {"type": "rating", "value": { - "id": 12345, - "author": {"title": "Иван"}, - "reviewTitle": "Хорошо", - "score": 5, - "modelExperience": "Живу в своей квартире", - "rated": "9 октября 2025", - "textSections": [ - {"title": "", "text": "Общий отзыв"}, - {"title": "Преимущества", "text": "Парк рядом"}, - {"title": "Недостатки", "text": "Парковки мало"}, - ], - }}, - {"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}}, - ]}, - }, - { - "type": "miniSerp", - "props": {"items": [ - { - "id": 7986882804, - "title": "3-к. квартира", - "description": "Описание", - "url": "/ekaterinburg/kvartiry/test_7986882804", - "price": "11 990 000 ₽", - "geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]}, - "seller": { - "name": "DOMRF66", - "type": "Компания", - "from": "На Авито с июля 2012", - "url": "/brands/domrf66", - "logos": [], + "data": { + "data": { + "page": { + "placeholders": [ + {"type": "breadcrumbs"}, + {"type": "title"}, + {"type": "gallery", "props": {}}, + { + "type": "housePage", + "props": { + "developmentData": { + "avitoId": 3171365, + "id": "MTE3LjM3MDg5MA", + "title": "Akademika Postovskogo 17a", + "address": "ул. Постовского, 17а", + "fullAddress": ( + "Свердловская обл., Екатеринбург, ул. Постовского, 17а" + ), + "coords": {"lat": 56.790699, "lng": 60.580191}, + "aboutDevelopment": { + "expandParams": { + "items": [ + { + "title": "Параметры дома", + "params": [ + {"type": "Год постройки", "value": "2019"}, + {"type": "Этажей", "value": "25"}, + {"type": "Тип дома", "value": "Монолитный"}, + {"type": "Пассажирский лифт", "value": "2"}, + {"type": "Консьерж", "value": "Да"}, + {"type": "Закрытый двор", "value": "Да"}, + {"type": "Детская площадка", "value": "Да"}, + ], + }, + ] + } + }, + "developer": {"name": "Атомстройкомплекс", "key": "atomstroy"}, + "mapPreview": { + "distance": "в 5 минутах ходьбы", + "objects": "школа, парк, аптека", + "pins": [{"lat": 56.79, "lng": 60.58, "type": "school"}], + }, + "ratingBadge": {"info": {"score": 4.666, "scoreString": "4,7"}}, + "ratingSummaryStat": { + "ratingStat": [ + {"score": 5, "count": 4}, + {"score": 4, "count": 2}, + ], + "reviewCount": 6, + }, + } + }, }, - }, - ]}, - }, - { - "type": "housePlacementHistory", - "props": {"items": [ - { - "id": 7000000001, - "title": "1-к. квартира", - "startPrice": 5000000, - "startPriceDate": 1697000000, - "lastPrice": 4900000, - "lastPriceDate": 1715000000, - "exposure": 210, - "itemImage": {}, - }, - ]}, - }, - {"type": "uxFeedback"}, - { - "type": "recommendations", - "props": {"items": [ - { - "title": "Похожая", - "priceRange": "от 6 млн", - "additionalInfo": ["Адрес 1"], - "url": "/rec1", - "images": [{"208x156": "https://avito.ru/img1.jpg"}], - }, - ]}, - }, - {"type": "padding"}, - ]}}} + { + "type": "reviews", + "props": { + "entries": [ + { + "type": "rating", + "value": { + "id": 12345, + "author": {"title": "Иван"}, + "reviewTitle": "Хорошо", + "score": 5, + "modelExperience": "Живу в своей квартире", + "rated": "9 октября 2025", + "textSections": [ + {"title": "", "text": "Общий отзыв"}, + {"title": "Преимущества", "text": "Парк рядом"}, + {"title": "Недостатки", "text": "Парковки мало"}, + ], + }, + }, + {"type": "pages", "value": {"nextPageUrl": "/reviews?page=2"}}, + ] + }, + }, + { + "type": "miniSerp", + "props": { + "items": [ + { + "id": 7986882804, + "title": "3-к. квартира", + "description": "Описание", + "url": "/ekaterinburg/kvartiry/test_7986882804", + "price": "11 990 000 ₽", + "geo": {"content": "Чкаловская, 2,7 км", "colors": ["#cf2734"]}, + "seller": { + "name": "DOMRF66", + "type": "Компания", + "from": "На Авито с июля 2012", + "url": "/brands/domrf66", + "logos": [], + }, + }, + ] + }, + }, + { + "type": "housePlacementHistory", + "props": { + "items": [ + { + "id": 7000000001, + "title": "1-к. квартира", + "startPrice": 5000000, + "startPriceDate": 1697000000, + "lastPrice": 4900000, + "lastPriceDate": 1715000000, + "exposure": 210, + "itemImage": {}, + }, + ] + }, + }, + {"type": "uxFeedback"}, + { + "type": "recommendations", + "props": { + "items": [ + { + "title": "Похожая", + "priceRange": "от 6 млн", + "additionalInfo": ["Адрес 1"], + "url": "/rec1", + "images": [{"208x156": "https://avito.ru/img1.jpg"}], + }, + ] + }, + }, + {"type": "padding"}, + ] + } + } + } } @@ -305,20 +336,28 @@ def test_recommendations() -> None: def test_missing_optional_widgets() -> None: """Парсер не падает если опциональные виджеты отсутствуют.""" state_minimal: dict = { - "data": {"data": {"page": {"placeholders": [ - { - "type": "housePage", - "props": {"developmentData": { - "avitoId": 9999, - "coords": {"lat": 56.0, "lng": 60.0}, - "aboutDevelopment": {"expandParams": {"items": []}}, - "developer": {}, - "mapPreview": {}, - "ratingBadge": {"info": {}}, - "ratingSummaryStat": {}, - }}, - }, - ]}}} + "data": { + "data": { + "page": { + "placeholders": [ + { + "type": "housePage", + "props": { + "developmentData": { + "avitoId": 9999, + "coords": {"lat": 56.0, "lng": 60.0}, + "aboutDevelopment": {"expandParams": {"items": []}}, + "developer": {}, + "mapPreview": {}, + "ratingBadge": {"info": {}}, + "ratingSummaryStat": {}, + } + }, + }, + ] + } + } + } } e = parse_houses_state(state_minimal, "/catalog/houses/test/9999") assert e.house.ext_id == 9999 @@ -338,9 +377,15 @@ def test_invalid_state_raises() -> None: def test_missing_house_page_raises() -> None: """ValueError если housePage виджет отсутствует.""" state: dict = { - "data": {"data": {"page": {"placeholders": [ - {"type": "breadcrumbs"}, - ]}}} + "data": { + "data": { + "page": { + "placeholders": [ + {"type": "breadcrumbs"}, + ] + } + } + } } with pytest.raises(ValueError, match="housePage"): parse_houses_state(state, "/catalog/houses/test/1") @@ -373,27 +418,42 @@ def test_strip_price() -> None: def test_mini_serp_no_seller() -> None: """Объявление без продавца не вызывает ошибку.""" state: dict = { - "data": {"data": {"page": {"placeholders": [ - { - "type": "housePage", - "props": {"developmentData": { - "avitoId": 111, - "coords": {"lat": 56.0, "lng": 60.0}, - "aboutDevelopment": {"expandParams": {"items": []}}, - "developer": {}, - "mapPreview": {}, - "ratingBadge": {"info": {}}, - "ratingSummaryStat": {}, - }}, - }, - { - "type": "miniSerp", - "props": {"items": [ - {"id": 12345, "title": "1-к.", "url": "/test", "price": "3 000 000 ₽", - "geo": {"content": "Центр", "colors": []}}, - ]}, - }, - ]}}} + "data": { + "data": { + "page": { + "placeholders": [ + { + "type": "housePage", + "props": { + "developmentData": { + "avitoId": 111, + "coords": {"lat": 56.0, "lng": 60.0}, + "aboutDevelopment": {"expandParams": {"items": []}}, + "developer": {}, + "mapPreview": {}, + "ratingBadge": {"info": {}}, + "ratingSummaryStat": {}, + } + }, + }, + { + "type": "miniSerp", + "props": { + "items": [ + { + "id": 12345, + "title": "1-к.", + "url": "/test", + "price": "3 000 000 ₽", + "geo": {"content": "Центр", "colors": []}, + }, + ] + }, + }, + ] + } + } + } } e = parse_houses_state(state, "/catalog/houses/test/111") assert len(e.mini_serp) == 1 @@ -458,3 +518,56 @@ def test_extract_preloaded_state_jsparse_fallback() -> None: def test_extract_preloaded_state_missing_returns_none() -> None: """Нет маркера → None (а не исключение).""" assert _extract_preloaded_state("no state here") is None + + +# --------------------------------------------------------------------------- +# Real-HTML fixture (regression for issue #1789 — MFE props nesting) +# --------------------------------------------------------------------------- +# +# Fixture: реальная страница /catalog/houses/ekaterinburg/ul_soni_morozovoy_190/572538, +# захвачена браузер-сервисом (2026-06-19). Avito перенёс данные виджетов на уровень +# глубже — под ключ с именем виджета: +# reviews: props['reviews']['entries'] (было props['entries']) +# miniSerp: props['miniSerp']['items'] (было props['items']) +# housePlacementHistory: props['housePlacementHistory']['items'] (было props['items']) +# recommendations: props['recommendations']['items'] (было props['items']) +# Из-за этого reviews/placement/seller-обогащение НЕ долетало в БД (0 строк). +# Этот тест ловит регрессию шейпа props. + + +def test_parse_mfe_fixture_widgets_reach_db() -> None: + """Фикстура MFE: reviews/placement/miniSerp/house_type/year_built распарсены непусто.""" + html = (FIXTURES_DIR / "avito_house_572538.html").read_text(encoding="utf-8") + state = _extract_preloaded_state(html) + assert state is not None, "URL-encoded __preloadedState__ должен распарситься" + + e = parse_houses_state(state, "/catalog/houses/ekaterinburg/ul_soni_morozovoy_190/572538") + + # housePage — тип дома и год постройки долетают + assert e.house.ext_id == 572538 + assert e.house.house_type is not None, "house_type должен парситься из expandParams" + assert e.house.house_type == "brick" + assert e.house.year_built == 1994 + assert e.house.reviews_count == 3 + + # reviews — >=1 отзыв с текстом, автором, оценкой + assert len(e.reviews) >= 1, "reviews должны распарситься из props['reviews']['entries']" + r = e.reviews[0] + assert r.ext_review_id, "у отзыва должен быть id" + assert r.author_name, "у отзыва должен быть автор" + assert r.score is not None, "у отзыва должна быть оценка" + assert r.text_main, "у отзыва должен быть основной текст (textSections)" + + # placement — >=1 item из props['housePlacementHistory']['items'] + assert len(e.placement_history) >= 1, "placement должен распарситься" + p = e.placement_history[0] + assert p.ext_item_id, "у placement item должен быть id" + assert p.start_price is not None + assert p.last_price is not None + assert isinstance(p.last_price_date, date) + + # miniSerp — >=1 объявление из props['miniSerp']['items'] + assert len(e.mini_serp) >= 1, "miniSerp должен распарситься" + m = e.mini_serp[0] + assert m.ext_item_id + assert m.price_rub is not None