From cd91d5e27909c5320600c734224e4d698163a212 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 20 Jun 2026 09:53:46 +0300 Subject: [PATCH] =?UTF-8?q?fix(scrapers):=20avito=20pipeline=20=E2=80=94?= =?UTF-8?q?=20=D0=BD=D0=BE=D0=B2=D0=BE=D1=81=D1=82=D1=80=D0=BE=D0=B9=D0=BA?= =?UTF-8?q?=D0=B8=20house=20graceful=20(=D0=B1=D0=B5=D0=B7=20housePage),?= =?UTF-8?q?=20AvitoBlockedError=20per-item=20=D0=BD=D0=B5=20=D0=BA=D1=80?= =?UTF-8?q?=D0=B0=D1=88=D0=B8=D1=82=20=D0=BF=D1=80=D0=BE=D0=B3=D0=BE=D0=BD?= =?UTF-8?q?=20(#1820)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/app/services/scrape_pipeline.py | 102 +++++++++--- .../app/services/scrapers/avito_houses.py | 135 +++++++++++++++- .../fixtures/avito_novostroyka_12879380.html | 5 + .../tests/scrapers/test_avito_anti_bot.py | 45 ++++-- .../backend/tests/test_avito_houses_parse.py | 148 +++++++++++++++++- .../backend/tests/test_scrape_pipeline.py | 138 ++++++++++++++++ 6 files changed, 533 insertions(+), 40 deletions(-) create mode 100644 tradein-mvp/backend/tests/fixtures/avito_novostroyka_12879380.html diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index cd25c417..56aa2299 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -86,6 +86,13 @@ _AVITO_HOUSES_BUDGET_S: float = 180.0 # бюджет на house-enrich фазу # явной деградации browser-сервиса, не мешает нормальным transient-ошибкам. _AVITO_DETAIL_CONSECUTIVE_TIMEOUT_ABORT: int = 5 +# #1820: порог ПОСЛЕДОВАТЕЛЬНЫХ блоков (AvitoBlockedError/AvitoRateLimitedError) внутри +# enrichment-фаз run_avito_pipeline. Один заблокированный фетч (house/detail) больше НЕ +# роняет весь прогон — блок ловится пер-item (как обычная ошибка), и лишь N подряд +# прерывают enrichment-фазу. Listings из SEARCH+SAVE сохраняются всегда. Зеркалит +# паттерн avito_detail_backfill (max_consecutive_blocks). +_AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT: int = 3 + # Default anchors ЕКБ — 5 точек покрытия города EKB_ANCHORS: list[tuple[float, float, str]] = [ (56.8400, 60.6050, "Центр"), @@ -181,11 +188,21 @@ async def run_avito_pipeline( shared_session — если передан, используется без закрытия (lifecycle у вызывающего). shared_browser — если передан в browser-mode, используется без закрытия. - AvitoBlockedError / AvitoRateLimitedError propagate наружу. + + Блокировки (AvitoBlockedError / AvitoRateLimitedError) в SEARCH-фазе (шаг 1) + по-прежнему propagate наружу (без листингов прогон бессмыслен). В enrichment-фазах + (houses шаг 4 + detail шаг 5 + house-detail шаг 5b) блок ловится ПЕР-ITEM как обычная + ошибка (#1820): один заблокированный фетч НЕ роняет весь прогон. Лишь + _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT блоков ПОДРЯД прерывают enrichment — но + PipelineResult всё равно возвращается с уже собранными listings/обогащениями. """ counters = PipelineCounters() lots: list[ScrapedLot] = [] detail_delay = request_delay_sec if request_delay_sec is not None else 7.0 + # #1820: общий счётчик ПОСЛЕДОВАТЕЛЬНЫХ блоков на все enrichment-фазы (4/5/5b). + # Сбрасывается на любом успешном fetch; при достижении порога — enrichment_aborted. + enrich_consecutive_blocks = 0 + enrichment_aborted = False browser_mode = settings.scraper_fetch_mode == "browser" session: AsyncSession | None = None @@ -261,7 +278,7 @@ async def run_avito_pipeline( # ── Step 4: enrich houses с shared session + sleep ────── touched_house_ids: set[int] = set() - if enrich_houses and unique_house_paths: + if enrich_houses and unique_house_paths and not enrichment_aborted: house_paths_list = list(unique_house_paths) for idx, house_path in enumerate(house_paths_list): try: @@ -270,12 +287,32 @@ async def run_avito_pipeline( ) house_counters = save_house_catalog_enrichment(db, enrichment) counters.houses_enriched += 1 + enrich_consecutive_blocks = 0 hid = house_counters.get("house_id") if hid: touched_house_ids.add(int(hid)) - except (AvitoBlockedError, AvitoRateLimitedError): - logger.error("pipeline:house BLOCKED at %s — propagating", house_path) - raise + except (AvitoBlockedError, AvitoRateLimitedError) as e: + # #1820: блок ловим пер-item как обычную ошибку — НЕ роняем прогон. + enrich_consecutive_blocks += 1 + counters.houses_failed += 1 + counters.errors.append(f"BLOCKED_HOUSE {house_path}: {e}") + logger.warning( + "pipeline:house BLOCKED at %s (consecutive=%d)", + house_path, + enrich_consecutive_blocks, + ) + if enrich_consecutive_blocks >= _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT: + logger.error( + "pipeline:enrich ABORT — %d consecutive blocks (IP rate-limited), " + "listings preserved", + enrich_consecutive_blocks, + ) + counters.errors.append( + f"AVITO_BLOCKED — enrichment aborted after " + f"{enrich_consecutive_blocks} consecutive blocks (house phase)" + ) + enrichment_aborted = True + break except Exception as e: logger.warning("pipeline:house_enrich failed for %s: %s", house_path, e) counters.houses_failed += 1 @@ -293,7 +330,7 @@ async def run_avito_pipeline( await asyncio.sleep(detail_delay) # ── Step 5: enrich detail для top-N priority listings ─── - if enrich_detail_top_n > 0: + if enrich_detail_top_n > 0 and not enrichment_aborted: priority_rows = ( db.execute( text(""" @@ -330,7 +367,6 @@ async def run_avito_pipeline( .all() ) - consecutive_blocks = 0 detail_house_paths: set[str] = set() for idx, row in enumerate(priority_rows): source_url: str = row["source_url"] @@ -350,28 +386,33 @@ async def run_avito_pipeline( hp = urlparse(enrichment_detail.house_catalog_url).path if hp: detail_house_paths.add(hp) - consecutive_blocks = 0 + enrich_consecutive_blocks = 0 except (AvitoBlockedError, AvitoRateLimitedError) as e: - consecutive_blocks += 1 + # #1820: блок ловим пер-item, НЕ роняем прогон. Счётчик общий с house-фазой. + enrich_consecutive_blocks += 1 counters.detail_failed += 1 counters.errors.append(f"BLOCKED_DETAIL #{idx}: {e}") logger.warning( "pipeline:detail BLOCKED #%d/%d (consecutive=%d): %s", idx + 1, len(priority_rows), - consecutive_blocks, + enrich_consecutive_blocks, e, ) - if consecutive_blocks >= 3: + if enrich_consecutive_blocks >= _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT: logger.error( - "pipeline:detail ABORT — %d consecutive blocks, IP rate-limited", - consecutive_blocks, + "pipeline:detail ABORT — %d consecutive blocks (IP rate-limited), " + "listings preserved", + enrich_consecutive_blocks, ) counters.errors.append( - f"AVITO_RATE_LIMITED — sweep aborted after " - f"{idx + 1}/{len(priority_rows)} details" + f"AVITO_BLOCKED — enrichment aborted after " + f"{idx + 1}/{len(priority_rows)} details (consecutive blocks)" ) - raise + # #1820: НЕ raise — прерываем только detail-фазу (и Step 5b), + # PipelineResult возвращается с уже сохранёнными listings/houses. + enrichment_aborted = True + break except Exception as e: counters.detail_failed += 1 counters.errors.append(f"detail {source_url}: {e}") @@ -398,7 +439,7 @@ async def run_avito_pipeline( # SERP-карточки больше не несут house-link (JS-render) → дома берём из # detail SSR. Дедуп против обработанных в Step 4 (unique_house_paths). new_house_paths = detail_house_paths - unique_house_paths - if enrich_houses and new_house_paths: + if enrich_houses and new_house_paths and not enrichment_aborted: counters.unique_houses += len(new_house_paths) nh_list = list(new_house_paths) for h_idx, house_path in enumerate(nh_list): @@ -408,14 +449,33 @@ async def run_avito_pipeline( ) house_counters = save_house_catalog_enrichment(db, enrichment) counters.houses_enriched += 1 + enrich_consecutive_blocks = 0 hid = house_counters.get("house_id") if hid: touched_house_ids.add(int(hid)) - except (AvitoBlockedError, AvitoRateLimitedError): - logger.error( - "pipeline:house(detail) BLOCKED at %s — propagating", house_path + except (AvitoBlockedError, AvitoRateLimitedError) as e: + # #1820: блок пер-item, общий счётчик; не роняем прогон. + enrich_consecutive_blocks += 1 + counters.houses_failed += 1 + counters.errors.append(f"BLOCKED_HOUSE(detail) {house_path}: {e}") + logger.warning( + "pipeline:house(detail) BLOCKED at %s (consecutive=%d)", + house_path, + enrich_consecutive_blocks, ) - raise + if enrich_consecutive_blocks >= _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT: + logger.error( + "pipeline:enrich ABORT — %d consecutive blocks " + "(IP rate-limited), listings preserved", + enrich_consecutive_blocks, + ) + counters.errors.append( + f"AVITO_BLOCKED — enrichment aborted after " + f"{enrich_consecutive_blocks} consecutive blocks " + f"(house-detail phase)" + ) + enrichment_aborted = True + break except Exception as e: logger.warning( "pipeline:house(detail) enrich failed for %s: %s", house_path, e diff --git a/tradein-mvp/backend/app/services/scrapers/avito_houses.py b/tradein-mvp/backend/app/services/scrapers/avito_houses.py index 787ddcfb..96d5cc22 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_houses.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_houses.py @@ -96,8 +96,16 @@ EXPAND_PARAMS_MAP: dict[str, tuple[str, str]] = { "Парковка": ("parking_type", "str"), "Детская площадка": ("has_playground", "bool_da"), "Закрытый двор": ("closed_yard", "bool_da"), + # Новостройки (developmentPage) — другие подписи параметров. Лифты во + # множественном числе; "Этажность" — диапазон ("от 4 до 24"), int-каст + # вернёт None (хранится только в raw_characteristics). + "Пассажирских лифтов": ("passenger_elevators", "int"), + "Грузовых лифтов": ("cargo_elevators", "int"), } +# URL-маркер новостройки: /catalog/novostroyki/... (vs вторичка /catalog/houses/...). +NOVOSTROYKA_URL_MARKER = "/catalog/novostroyki/" + # Нормализация типа дома из Avito текста в DB enum HOUSE_TYPE_MAP: dict[str, str] = { "монолитный": "monolith", @@ -413,6 +421,79 @@ def _parse_house_page(widget: dict[str, Any]) -> HouseInfo: ) +def _is_novostroyka_url(house_url: str) -> bool: + """True если house_url — страница новостройки (/catalog/novostroyki/...). + + Структура новостройки отличается от вторички: вместо виджета 'housePage' + используется 'developmentPage', нет отзывов/истории размещений. + """ + return NOVOSTROYKA_URL_MARKER in house_url + + +def _parse_development_page(widget: dict[str, Any]) -> HouseInfo: + """Парсит виджет 'developmentPage' (новостройка) → HouseInfo. + + Структура отличается от 'housePage' (вторичка): + - данные под props['developmentPage']['developmentData'] (fallback props['developmentData']) + - характеристики не в developmentData.aboutDevelopment.expandParams.items, + а в aboutDevelopment.tabs[*].expandParams.items[*].params (плоский список params). + - НЕТ рейтинга (ratingBadge/ratingSummaryStat), года постройки (дом строится — + есть "Срок сдачи"), отзывов, истории размещений. + Извлекаем то, что есть: id/адрес/координаты/застройщик/инфраструктура + тип дома, + этажность (диапазон → raw_characteristics), лифты из таблицы "Параметры". + """ + props = widget.get("props", {}) + dd = props.get("developmentPage", {}).get("developmentData") or props.get("developmentData", {}) + + # Характеристики новостройки лежат в tabs[*].expandParams.items[*].params. + # Собираем их в форму, совместимую с _parse_expand_params (category → params). + char_categories: list[dict[str, Any]] = [] + about = dd.get("aboutDevelopment", {}) + for tab in about.get("tabs", []): + if not isinstance(tab, dict): + continue + items = tab.get("expandParams", {}).get("items", []) + for item in items: + if isinstance(item, dict) and isinstance(item.get("params"), list): + char_categories.append(item) + parsed_params = _parse_expand_params(char_categories) + + map_preview = dd.get("mapPreview", {}) + developer = dd.get("developer", {}) + coords = dd.get("coords", {}) + + return HouseInfo( + ext_id=dd.get("avitoId", 0), + ext_id_hash=dd.get("id"), + title=dd.get("title"), + short_address=dd.get("address"), + full_address=dd.get("fullAddress") or None, + lat=coords.get("lat"), + lon=coords.get("lng"), + year_built=parsed_params.get("year_built"), # обычно None — дом строится + total_floors=parsed_params.get("total_floors"), + house_type=parsed_params.get("house_type"), + material_floors=parsed_params.get("material_floors"), + hot_water=parsed_params.get("hot_water"), + passenger_elevators=parsed_params.get("passenger_elevators"), + cargo_elevators=parsed_params.get("cargo_elevators"), + has_concierge=parsed_params.get("has_concierge"), + parking_type=parsed_params.get("parking_type"), + has_playground=parsed_params.get("has_playground"), + closed_yard=parsed_params.get("closed_yard"), + developer_name=developer.get("name"), + developer_key=developer.get("key"), + infrastructure_summary=map_preview.get("objects"), + infrastructure_walk_distance=map_preview.get("distance"), + rating_score=None, + rating_string=None, + reviews_count=None, + rating_distribution=[], + map_pins=map_preview.get("pins", []), + raw_characteristics=char_categories, + ) + + def _parse_single_review(r: dict[str, Any]) -> HouseReview: """Парсит один объект отзыва из entries.""" text_main: str | None = None @@ -626,14 +707,61 @@ def _extract_preloaded_state(html: str) -> dict[str, Any] | None: return None +def _parse_development_or_skip( + placeholders: list[dict[str, Any]], house_url: str +) -> HouseCatalogEnrichment: + """Обрабатывает страницу без виджета 'housePage' (новостройка или неизвестная форма). + + - Есть виджет 'developmentPage' → парсим его (partial: дом/застройщик/тип/лифты, + без отзывов/истории/рейтинга). + - Нет developmentPage, но URL — новостройка → graceful skip: warning + минимальное + обогащение (ext_id из URL, без характеристик). НЕ кидаем ValueError, чтобы один + незнакомый шейп новостройки не валил весь enrichment-loop pipeline. + - Иначе (вторичка без housePage) → ValueError (регрессия шейпа вторички). + """ + dev_widget = _get_widget(placeholders, "developmentPage") + if dev_widget is not None: + house_info = _parse_development_page(dev_widget) + logger.info( + "developmentPage (новостройка) распарсен: ext_id=%s type=%s url=%s", + house_info.ext_id, + house_info.house_type, + house_url, + ) + return HouseCatalogEnrichment(house_url=house_url, house=house_info) + + if _is_novostroyka_url(house_url): + logger.warning( + "Новостройка без виджета 'developmentPage'/'housePage' (%s) — " + "graceful skip, обогащение пропущено", + house_url, + ) + ext_id = _ext_id_from_url(house_url) + return HouseCatalogEnrichment(house_url=house_url, house=HouseInfo(ext_id=ext_id)) + + raise ValueError("Виджет 'housePage' не найден в placeholders") + + +def _ext_id_from_url(house_url: str) -> int: + """Извлекает числовой avitoId из хвоста house_url (...//). 0 если нет.""" + m = re.search(r"/(\d+)/?(?:\?|$)", house_url) + return int(m.group(1)) if m else 0 + + def parse_houses_state(state: dict[str, Any], house_url: str) -> HouseCatalogEnrichment: """Парсит уже разобранный __preloadedState__ dict → HouseCatalogEnrichment. Pure функция: нет HTTP, нет I/O. Принимает state dict и house_url (для поля HouseCatalogEnrichment.house_url). Используется в тестах и из fetch_house_catalog. + Новостройки (/catalog/novostroyki/...) не имеют виджета 'housePage' — вместо него + 'developmentPage'. Для них парсим что есть (id/адрес/застройщик/тип дома/лифты) и + возвращаем partial-обогащение без отзывов/истории. Если структура совсем не + распознана — graceful: warning + минимальное обогащение (НЕ ValueError). + Raises: - ValueError если placeholders не массив или housePage виджет не найден. + ValueError если placeholders не массив, или (только для вторички) housePage + виджет не найден. """ try: raw_placeholders: list[dict[str, Any]] = state["data"]["data"]["page"]["placeholders"] @@ -649,10 +777,11 @@ def parse_houses_state(state: dict[str, Any], house_url: str) -> HouseCatalogEnr # раскрываем в плоский список (поддержка старой и новой формы). placeholders = _flatten_widgets(raw_placeholders) - # housePage — обязательный виджет + # housePage — обязательный виджет для вторички. Новостройки используют + # 'developmentPage' и не имеют housePage — обрабатываем отдельно (graceful). house_page_widget = _get_widget(placeholders, "housePage") if house_page_widget is None: - raise ValueError("Виджет 'housePage' не найден в placeholders") + return _parse_development_or_skip(placeholders, house_url) house_info = _parse_house_page(house_page_widget) diff --git a/tradein-mvp/backend/tests/fixtures/avito_novostroyka_12879380.html b/tradein-mvp/backend/tests/fixtures/avito_novostroyka_12879380.html new file mode 100644 index 00000000..b0cc846c --- /dev/null +++ b/tradein-mvp/backend/tests/fixtures/avito_novostroyka_12879380.html @@ -0,0 +1,5 @@ + + + + diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py index 3cc12537..b2b36e31 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py @@ -175,8 +175,17 @@ async def test_fetch_around_raises_rate_limited_on_429() -> None: @pytest.mark.asyncio async def test_pipeline_aborts_after_3_consecutive_blocks() -> None: - """Pipeline должен прервать sweep после 3 consecutive 403 (re-raise).""" - from app.services.scrape_pipeline import run_avito_pipeline + """#1820: 3 consecutive 403 в detail-фазе прерывают enrichment, но НЕ роняют прогон. + + Раньше pipeline re-raise'ил AvitoBlockedError (один блок ронял весь прогон). Теперь + блок ловится пер-item, и лишь N consecutive блоков прерывают enrichment-фазу — + PipelineResult возвращается, detail_failed считается, лишние detail-фетчи не делаются. + """ + from app.services.scrape_pipeline import ( + _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT, + PipelineResult, + run_avito_pipeline, + ) mock_db = MagicMock() mock_scraper = MagicMock() @@ -196,28 +205,34 @@ async def test_pipeline_aborts_after_3_consecutive_blocks() -> None: mock_db.execute.return_value = mock_execute block_error = AvitoBlockedError("HTTP 403") + detail_mock = AsyncMock(side_effect=block_error) with ( patch("app.services.scrape_pipeline.AvitoScraper", return_value=mock_scraper), - patch( - "app.services.scrape_pipeline.fetch_detail", - AsyncMock(side_effect=block_error), - ), + patch("app.services.scrape_pipeline.fetch_detail", detail_mock), patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()), patch( "curl_cffi.requests.AsyncSession", return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()), ), ): - with pytest.raises(AvitoBlockedError): - await run_avito_pipeline( - mock_db, - 56.8, - 60.6, - 1000, - enrich_houses=False, - enrich_detail_top_n=4, - ) + # НЕ должно бросать наружу + result = await run_avito_pipeline( + mock_db, + 56.8, + 60.6, + 1000, + enrich_houses=False, + enrich_detail_top_n=4, + ) + + assert isinstance(result, PipelineResult) + # enrichment прерван ровно после порога consecutive блоков (4-я строка не фетчилась) + assert detail_mock.await_count == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT + assert result.counters.detail_attempted == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT + assert result.counters.detail_failed == _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT + # в errors зафиксирован abort + assert any("AVITO_BLOCKED" in e for e in result.counters.errors) @pytest.mark.asyncio diff --git a/tradein-mvp/backend/tests/test_avito_houses_parse.py b/tradein-mvp/backend/tests/test_avito_houses_parse.py index e40a13f2..26febb3d 100644 --- a/tradein-mvp/backend/tests/test_avito_houses_parse.py +++ b/tradein-mvp/backend/tests/test_avito_houses_parse.py @@ -375,7 +375,7 @@ def test_invalid_state_raises() -> None: def test_missing_house_page_raises() -> None: - """ValueError если housePage виджет отсутствует.""" + """ValueError если housePage виджет отсутствует (вторичка — /catalog/houses/).""" state: dict = { "data": { "data": { @@ -391,6 +391,152 @@ def test_missing_house_page_raises() -> None: parse_houses_state(state, "/catalog/houses/test/1") +def test_novostroyka_minimal_development_page() -> None: + """Новостройка: виджет 'developmentPage' парсится в partial HouseInfo без ValueError.""" + state: dict = { + "data": { + "data": { + "page": { + "placeholders": [ + {"type": "reNavigation"}, + { + "type": "developmentPage", + "props": { + "developmentPage": { + "developmentData": { + "avitoId": 12879380, + "id": "MGDKHN70PQ", + "title": "ЖК Береговой", + "address": "ул. Щербакова, жилые дома", + "fullAddress": "", + "coords": {"lat": 56.77005, "lng": 60.67244}, + "developer": {"name": "Талан", "key": "talan-1"}, + "mapPreview": { + "distance": "в 15 минутах ходьбы", + "objects": "школа, детский сад", + "pins": [{"lat": 56.77, "lng": 60.67}], + }, + "aboutDevelopment": { + "tabs": [ + { + "title": "Параметры", + "expandParams": { + "items": [ + { + "params": [ + { + "type": "Тип дома", + "value": "Монолит", + }, + { + "type": "Этажность", + "value": "от 4 до 24", + }, + { + "type": ( + "Пассажирских лифтов" + ), + "value": "42", + }, + { + "type": "Грузовых лифтов", + "value": "60", + }, + ] + } + ] + }, + } + ] + }, + } + } + }, + }, + {"type": "uxFeedback"}, + ] + } + } + } + } + url = "/catalog/novostroyki/ekaterinburg/zhk_beregovoy/12879380" + e = parse_houses_state(state, url) + + assert isinstance(e, HouseCatalogEnrichment) + assert e.house_url == url + h = e.house + assert h.ext_id == 12879380 + assert h.ext_id_hash == "MGDKHN70PQ" + assert h.title == "ЖК Береговой" + assert h.short_address == "ул. Щербакова, жилые дома" + assert h.lat == 56.77005 + assert h.lon == 60.67244 + assert h.house_type == "monolith" # "Монолит" → monolith + assert h.passenger_elevators == 42 + assert h.cargo_elevators == 60 + # Этажность = диапазон "от 4 до 24" → int-каст None (только в raw_characteristics) + assert h.total_floors is None + assert h.year_built is None # дом строится, года постройки нет + assert h.developer_name == "Талан" + assert h.developer_key == "talan-1" + assert h.infrastructure_summary == "школа, детский сад" + assert h.infrastructure_walk_distance == "в 15 минутах ходьбы" + assert len(h.map_pins) == 1 + # Нет рейтинга/отзывов/истории у новостройки + assert h.rating_score is None + assert e.reviews == [] + assert e.placement_history == [] + assert e.recommendations == [] + + +def test_novostroyka_no_development_page_graceful_skip() -> None: + """Новостройка без developmentPage/housePage → graceful skip (НЕ ValueError).""" + state: dict = { + "data": { + "data": { + "page": { + "placeholders": [ + {"type": "reNavigation"}, + {"type": "uxFeedback"}, + ] + } + } + } + } + url = "/catalog/novostroyki/ekaterinburg/zhk_unknown/55667788" + e = parse_houses_state(state, url) + + assert isinstance(e, HouseCatalogEnrichment) + assert e.house_url == url + # ext_id извлечён из хвоста URL + assert e.house.ext_id == 55667788 + assert e.reviews == [] + assert e.placement_history == [] + + +def test_novostroyka_real_fixture_end_to_end() -> None: + """Реальная фикстура новостройки: parse не кидает ValueError, отдаёт partial-обогащение.""" + html = (FIXTURES_DIR / "avito_novostroyka_12879380.html").read_text(encoding="utf-8") + state = _extract_preloaded_state(html) + assert state is not None, "URL-encoded __preloadedState__ должен распарситься" + + url = "/catalog/novostroyki/ekaterinburg/semeynyy_kvartal_beregovoy/12879380" + e = parse_houses_state(state, url) + + assert isinstance(e, HouseCatalogEnrichment) + h = e.house + assert h.ext_id == 12879380 + assert h.ext_id_hash # base64 internal id + assert h.house_type == "monolith" + assert h.passenger_elevators is not None and h.passenger_elevators > 0 + assert h.lat is not None and h.lon is not None + assert h.developer_name # застройщик присутствует + # Новостройка: нет отзывов/истории/года постройки + assert h.year_built is None + assert e.reviews == [] + assert e.placement_history == [] + + def test_house_type_normalization() -> None: """Все варианты типа дома нормализуются корректно.""" from app.services.scrapers.avito_houses import _normalize_house_type diff --git a/tradein-mvp/backend/tests/test_scrape_pipeline.py b/tradein-mvp/backend/tests/test_scrape_pipeline.py index 84faf283..8cdb2a5e 100644 --- a/tradein-mvp/backend/tests/test_scrape_pipeline.py +++ b/tradein-mvp/backend/tests/test_scrape_pipeline.py @@ -6,11 +6,13 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest from app.services.scrape_pipeline import ( + _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT, PipelineCounters, PipelineResult, run_avito_full_load, run_avito_pipeline, ) +from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError from app.services.scrapers.base import ScrapedLot @@ -130,3 +132,139 @@ async def test_full_load_no_incremental_days_since_none() -> None: kwargs = scraper.fetch_all_secondary.call_args.kwargs assert kwargs["since"] is None + + +# ── #1820: AvitoBlockedError per-item не роняет весь pipeline ───────────────── + + +def _block_test_scraper(lots: list[ScrapedLot]) -> MagicMock: + scraper = MagicMock() + scraper.__aenter__ = AsyncMock(return_value=scraper) + scraper.__aexit__ = AsyncMock(return_value=None) + scraper.fetch_around = AsyncMock(return_value=lots) + scraper._cffi = None + return scraper + + +@pytest.mark.asyncio +async def test_pipeline_single_house_block_does_not_crash() -> None: + """#1820: один заблокированный house-фетч НЕ роняет прогон. + + Listings из SEARCH+SAVE сохранены, блок зафиксирован в counters (houses_failed/errors), + остальные дома обогащаются дальше (enrichment не прерван < порога). + """ + mock_db = MagicMock() + lots = [ + _make_lot("1", "https://www.avito.ru/catalog/houses/ekb/a/100"), + _make_lot("2", "https://www.avito.ru/catalog/houses/ekb/b/200"), + ] + scraper = _block_test_scraper(lots) + + # 1-й house блокируется, 2-й — успех. + fetch_house = AsyncMock(side_effect=[AvitoBlockedError("HTTP 403"), MagicMock()]) + + with ( + patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper), + patch("app.services.scrape_pipeline.save_listings", return_value=(2, 0)), + patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house), + patch( + "app.services.scrape_pipeline.save_house_catalog_enrichment", + return_value={"house_id": 42}, + ), + patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()), + patch( + "curl_cffi.requests.AsyncSession", + return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()), + ), + ): + result = await run_avito_pipeline( + mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0 + ) + + assert isinstance(result, PipelineResult) + # listings сохранены всегда + assert result.counters.lots_inserted == 2 + # один дом упал по блоку, второй обогащён + assert result.counters.houses_failed == 1 + assert result.counters.houses_enriched == 1 + assert fetch_house.await_count == 2 # enrichment НЕ прерван — оба дома попробованы + assert any("BLOCKED_HOUSE" in e for e in result.counters.errors) + assert 42 in result.touched_house_ids + + +@pytest.mark.asyncio +async def test_pipeline_consecutive_house_blocks_abort_but_return_result() -> None: + """#1820: N consecutive house-блоков прерывают enrichment, но PipelineResult возвращён.""" + mock_db = MagicMock() + n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT + # Создаём n+2 дома — после n блоков подряд enrichment прерывается. + lots = [ + _make_lot(str(i), f"https://www.avito.ru/catalog/houses/ekb/h{i}/{100 + i}") + for i in range(n + 2) + ] + scraper = _block_test_scraper(lots) + + fetch_house = AsyncMock(side_effect=AvitoRateLimitedError("HTTP 429")) + + with ( + patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper), + patch("app.services.scrape_pipeline.save_listings", return_value=(len(lots), 0)), + patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house), + patch("app.services.scrape_pipeline.save_house_catalog_enrichment", return_value={}), + patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()), + patch( + "curl_cffi.requests.AsyncSession", + return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()), + ), + ): + result = await run_avito_pipeline( + mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=0 + ) + + assert isinstance(result, PipelineResult) + assert result.counters.lots_inserted == len(lots) # listings сохранены + # enrichment прерван ровно после порога — лишние дома НЕ фетчились + assert fetch_house.await_count == n + assert result.counters.houses_failed == n + assert any("AVITO_BLOCKED" in e for e in result.counters.errors) + + +@pytest.mark.asyncio +async def test_pipeline_detail_block_aborts_skips_step5b() -> None: + """#1820: consecutive detail-блоки прерывают detail-фазу И Step 5b (houses из detail).""" + mock_db = MagicMock() + n = _AVITO_PIPELINE_CONSECUTIVE_BLOCK_ABORT + scraper = _block_test_scraper([]) # SERP пуст → сразу detail-фаза по priority_rows + + # priority_rows: n+1 строк, все блокируются в detail. + row = MagicMock() + row.__getitem__ = lambda self, key: ( + "https://www.avito.ru/ekaterinburg/kvartiry/test-1" if key == "source_url" else None + ) + mappings = MagicMock() + mappings.all.return_value = [row] * (n + 1) + execute = MagicMock() + execute.mappings.return_value = mappings + mock_db.execute.return_value = execute + + fetch_detail = AsyncMock(side_effect=AvitoBlockedError("HTTP 403")) + fetch_house = AsyncMock() # должен НЕ вызываться (Step 5b пропущен) + + with ( + patch("app.services.scrape_pipeline.AvitoScraper", return_value=scraper), + patch("app.services.scrape_pipeline.fetch_detail", fetch_detail), + patch("app.services.scrape_pipeline.fetch_house_catalog", fetch_house), + patch("app.services.scrape_pipeline.asyncio.sleep", AsyncMock()), + patch( + "curl_cffi.requests.AsyncSession", + return_value=AsyncMock(__aenter__=AsyncMock(), __aexit__=AsyncMock()), + ), + ): + result = await run_avito_pipeline( + mock_db, 56.8, 60.6, 1000, enrich_houses=True, enrich_detail_top_n=n + 1 + ) + + assert isinstance(result, PipelineResult) + assert fetch_detail.await_count == n # detail прерван на пороге + assert result.counters.detail_failed == n + assert fetch_house.await_count == 0 # Step 5b пропущен (enrichment_aborted) -- 2.45.3