From e65b4d44d545048994a113d70076f165952372be Mon Sep 17 00:00:00 2001 From: Light1YT Date: Thu, 25 Jun 2026 11:45:16 +0500 Subject: [PATCH] fix(avito): graceful sidecar HTTP 500/503 in browser SERP fetch (#1790) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _fetch_serp_html_browser вызывал BrowserFetcher.fetch без обёртки, поэтому httpx.HTTPStatusError от tradein-browser sidecar (500 NS_ERROR_PROXY_TOO_MANY_REQUESTS / Page.goto Timeout, 503 browser-unavailable) всплывал в generic except run_avito_full_load → mark_failed + re-raise, роняя весь обход после тысяч инкрементально сохранённых карточек (прод run #356: 3186 сохранено, один поздний 500 → весь run failed). Классифицируем тело sidecar-ошибки и роутим в существующую rotation/retry-машинерию: soft-ban (too_many_requests / 503 / browser unavailable) → ротация exit-IP; transient (timeout / прочие 5xx) → bounded backoff-retry без ротации; оба бюджета исчерпаны → AvitoRateLimitedError, чтобы run ушёл в graceful mark_banned (partial preserved, scheduler retry next cycle), а не mark_failed. Часть 1 issue #1790 (AVITO_PROXY_MAX_ROTATIONS=0→4) уже исправлена на проде; это browser-resilience follow-up. Refs #1790 --- .../backend/app/services/scrapers/avito.py | 89 +++++++++- .../backend/tests/test_avito_browser_mode.py | 161 +++++++++++++++++- 2 files changed, 248 insertions(+), 2 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index 9f4de0df..9e8ec7f6 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -27,6 +27,7 @@ import asyncio import inspect import logging import math +import random import re from collections.abc import Callable from datetime import date, datetime, timedelta, timezone @@ -272,6 +273,18 @@ _AVITO_429_RECONNECT_RETRIES = 3 _AVITO_403_MAX_RETRIES = 5 _AVITO_403_BACKOFF_SEC = 2.0 +# tradein-browser sidecar (POST /fetch) отдаёт HTTP 500 на ЛЮБУЮ внутреннюю +# ошибку страницы (тело {"error": "ErrorType: message"}) и HTTP 503 когда браузер +# не поднимается (прокси упал). Транзиентные сбои (Page.goto Timeout, Target +# closed, прочие NS_ERROR_/5xx) лечатся коротким backoff-retry того же url БЕЗ +# ротации IP; soft-ban (NS_ERROR_PROXY_TOO_MANY_REQUESTS / 503 / browser +# unavailable) — ротацией exit-IP. Когда ОБА бюджета исчерпаны → AvitoRateLimitedError, +# чтобы run_avito_full_load/run_avito_city_sweep ушли в graceful mark_banned путь +# (partial preserved), а не в generic except → mark_failed (прод-инцидент run #356: +# один поздний 500 ронял весь обход после 3186 сохранённых объявлений). +_AVITO_SIDECAR_TRANSIENT_RETRIES = 2 +_AVITO_SIDECAR_TRANSIENT_BACKOFF_SEC = 2.0 + # Per-bucket block tolerance в fetch_all_secondary: одиночный 429/блок на ОДНОМ # (комнатность × ценовой брекет) бакете (напр. deep-pagination dense-бакета) НЕ должен # ронять весь sweep — бакет скипается, обход продолжается. Только N ПОДРЯД заблокированных @@ -453,12 +466,86 @@ class AvitoScraper(BaseScraper): доступен только если _cffi поднят (standalone __aenter__); в pipeline-режиме с shared-browser _cffi=None → fallback пропускается. 3. curl-fallback тоже firewall/пусто (или недоступен) → AvitoBlockedError. + + Отдельная ветка — sidecar HTTP-ошибка (#356): BrowserFetcher пробрасывает + httpx.HTTPStatusError (500/503), если /fetch упал внутри (Page.goto Timeout, + NS_ERROR_PROXY_TOO_MANY_REQUESTS, браузер не поднялся). Раньше она всплывала + неперехваченной → generic except в run_avito_full_load → mark_failed + + re-raise, что роняло весь обход даже после тысяч инкрементально сохранённых + карточек. Теперь классифицируем тело ошибки и роутим в ту же + ротация+retry-машинерию: soft-ban → ротация IP, transient → bounded backoff, + оба бюджета исчерпаны → AvitoRateLimitedError (graceful mark_banned). """ assert self._browser is not None max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0 rot_done = 0 + # Независимый бюджет коротких retry на transient sidecar-ошибки (timeout/ + # crash) — НЕ тратит бюджет ротации. Каждая ветка ниже (ротация / transient) + # уменьшает свой счётчик → цикл всегда движется к условию исчерпания. + transient_left = _AVITO_SIDECAR_TRANSIENT_RETRIES while True: - html = await self._browser.fetch(url) + try: + html = await self._browser.fetch(url) + except httpx.HTTPStatusError as exc: + # Тело sidecar-ошибки: {"error": "ErrorType: message"}. Парсим + # устойчиво — невалидный JSON не должен ронять обработку. + status = exc.response.status_code + try: + error_text = str(exc.response.json().get("error", "")) + except Exception: + error_text = exc.response.text + lowered = error_text.lower() + + is_soft_ban = ( + "too_many_requests" in lowered + or status == 503 + or "browser unavailable" in lowered + or "proxy may be down" in lowered + ) + if is_soft_ban: + # Soft proxy rate-limit / прокси недоступен → свежий exit-IP. + if rot_done < max_rot and await self._rotate_ip(): + rot_done += 1 + logger.info( + "avito page=%d sidecar soft-ban (status=%d) — " + "retry after rotation #%d: %s", + page, + status, + rot_done, + error_text[:200], + ) + continue + else: + # Transient (Page.goto Timeout / Target closed / прочие 5xx) → + # короткий backoff-retry того же url БЕЗ ротации. + if transient_left > 0: + transient_left -= 1 + backoff = _AVITO_SIDECAR_TRANSIENT_BACKOFF_SEC + random.uniform(0, 0.5) + logger.warning( + "avito page=%d sidecar transient (status=%d) — " + "retry in %.1fs (left=%d): %s", + page, + status, + backoff, + transient_left, + error_text[:200], + ) + await asyncio.sleep(backoff) + continue + # Оба бюджета (ротация + transient) исчерпаны → graceful soft-ban, + # чтобы run_avito_full_load/city_sweep ушли в mark_banned (partial + # preserved), а НЕ в generic except → mark_failed. Намеренно + # AvitoRateLimitedError, не AvitoBlockedError и не bare exception. + logger.error( + "avito page=%d sidecar error budget exhausted (status=%d) url=%s: %s", + page, + status, + url, + error_text[:200], + ) + raise AvitoRateLimitedError( + f"avito SERP browser-sidecar error (page={page}): {error_text[:200]}" + ) from exc if not _is_firewall_page(html): return html if rot_done < max_rot and await self._rotate_ip(): diff --git a/tradein-mvp/backend/tests/test_avito_browser_mode.py b/tradein-mvp/backend/tests/test_avito_browser_mode.py index 3959f9fb..95f3eb74 100644 --- a/tradein-mvp/backend/tests/test_avito_browser_mode.py +++ b/tradein-mvp/backend/tests/test_avito_browser_mode.py @@ -15,12 +15,13 @@ from __future__ import annotations import os from unittest.mock import AsyncMock, MagicMock, patch +import httpx import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper -from app.services.scrapers.avito_exceptions import AvitoBlockedError +from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError from app.services.scrapers.base import ScrapedLot # --------------------------------------------------------------------------- @@ -31,6 +32,18 @@ _NORMAL_HTML = "normal avito SERP" _FIREWALL_HTML = f"{_FIREWALL_MARKERS[0]}" +def _sidecar_error(status: int, error: str) -> httpx.HTTPStatusError: + """Build a tradein-browser sidecar HTTPStatusError with a real JSON body. + + Mirrors what BrowserFetcher.fetch re-raises: an httpx.HTTPStatusError whose + `.response` is intact so `.response.json()["error"]` and `.response.status_code` + work the same as in production. + """ + request = httpx.Request("POST", "http://tradein-browser:3000/fetch") + response = httpx.Response(status, json={"error": error}, request=request) + return httpx.HTTPStatusError(f"{status} error", request=request, response=response) + + def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock: """Return a mock BrowserFetcher context manager that yields html from fetch().""" mock = MagicMock() @@ -358,3 +371,149 @@ async def test_browser_mode_rotation_returns_false_raises_blocked() -> None: with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)): with pytest.raises(AvitoBlockedError, match="browser-mode"): await scraper._fetch_serp_html("https://avito.ru/test", page=1) + + +# --------------------------------------------------------------------------- +# 6. #356 — sidecar HTTPStatusError (500/503) → graceful AvitoRateLimitedError +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_sidecar_proxy_too_many_requests_rotates_then_succeeds() -> None: + """#356: sidecar 500 NS_ERROR_PROXY_TOO_MANY_REQUESTS, rotation available → + after IP rotation a retry succeeds and the clean HTML is returned.""" + scraper = AvitoScraper() + + mock_browser = MagicMock() + # First fetch raises the soft-ban 500; after rotation the second fetch is clean. + mock_browser.fetch = AsyncMock( + side_effect=[ + _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"), + _NORMAL_HTML, + ] + ) + scraper._browser = mock_browser + mock_rotate = AsyncMock(return_value=True) + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" + mock_settings.avito_proxy_max_rotations = 2 + with patch.object(scraper, "_rotate_ip", new=mock_rotate): + result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=3) + + assert result == _NORMAL_HTML + mock_rotate.assert_awaited_once() + assert mock_browser.fetch.await_count == 2 + + +@pytest.mark.asyncio +async def test_sidecar_proxy_too_many_requests_exhausted_raises_rate_limited() -> None: + """#356: sidecar 500 proxy-too-many with rotation budget exhausted → + AvitoRateLimitedError (NOT a bare HTTPStatusError, NOT AvitoBlockedError).""" + scraper = AvitoScraper() + + mock_browser = MagicMock() + mock_browser.fetch = AsyncMock( + side_effect=_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS") + ) + scraper._browser = mock_browser + mock_rotate = AsyncMock(return_value=True) + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" + mock_settings.avito_proxy_max_rotations = 2 + with patch.object(scraper, "_rotate_ip", new=mock_rotate): + with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): + await scraper._fetch_serp_html("https://www.avito.ru/test", page=3) + + # 2 rotations consumed + 1 final fetch that exhausts the budget. + assert mock_rotate.await_count == 2 + assert mock_browser.fetch.await_count == 3 + + +@pytest.mark.asyncio +async def test_sidecar_transient_timeout_exhausts_budget_no_infinite_loop() -> None: + """#356: sidecar 500 transient (Timeout) repeated past the transient budget with + NO rotations → AvitoRateLimitedError after bounded retries (no infinite loop).""" + scraper = AvitoScraper() + + mock_browser = MagicMock() + mock_browser.fetch = AsyncMock( + side_effect=_sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded") + ) + scraper._browser = mock_browser + mock_rotate = AsyncMock(return_value=True) + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.avito_proxy_rotate_url = None # no rotation → transient budget only + mock_settings.avito_proxy_max_rotations = 0 + # Patch sleep so the bounded backoff doesn't slow the test. + with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): + with patch.object(scraper, "_rotate_ip", new=mock_rotate): + with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): + await scraper._fetch_serp_html("https://www.avito.ru/test", page=5) + + # Transient is NOT a soft-ban → never rotates. + mock_rotate.assert_not_awaited() + # Bounded: _AVITO_SIDECAR_TRANSIENT_RETRIES (2) retries + final attempt = 3. + assert mock_browser.fetch.await_count == 3 + + +@pytest.mark.asyncio +async def test_sidecar_503_browser_unavailable_exhausted_raises_rate_limited() -> None: + """#356: sidecar 503 'browser unavailable (proxy may be down)' with rotation + exhausted → AvitoRateLimitedError (soft-ban classification via status==503).""" + scraper = AvitoScraper() + + mock_browser = MagicMock() + mock_browser.fetch = AsyncMock( + side_effect=_sidecar_error(503, "browser unavailable (proxy may be down)") + ) + scraper._browser = mock_browser + mock_rotate = AsyncMock(return_value=True) + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" + mock_settings.avito_proxy_max_rotations = 1 + with patch.object(scraper, "_rotate_ip", new=mock_rotate): + with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): + await scraper._fetch_serp_html("https://www.avito.ru/test", page=7) + + # 1 rotation consumed then exhausted. + assert mock_rotate.await_count == 1 + assert mock_browser.fetch.await_count == 2 + + +@pytest.mark.asyncio +async def test_sidecar_transient_then_soft_ban_independent_budgets() -> None: + """#356: смешанная последовательность transient → soft-ban пинит, что бюджеты + независимы и корректно чередуются в одном while-цикле. + + attempt 1: transient Timeout → тратит transient_left (1 из 2), backoff, continue. + attempt 2: too_many_requests → soft-ban, тратит ротацию (1 из 1), continue. + attempt 3: too_many_requests, ротация исчерпана → AvitoRateLimitedError. + """ + scraper = AvitoScraper() + + mock_browser = MagicMock() + mock_browser.fetch = AsyncMock( + side_effect=[ + _sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded"), + _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"), + _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"), + ] + ) + scraper._browser = mock_browser + mock_rotate = AsyncMock(return_value=True) + + with patch("app.services.scrapers.avito.settings") as mock_settings: + mock_settings.avito_proxy_rotate_url = "http://proxy/rotate" + mock_settings.avito_proxy_max_rotations = 1 + with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): + with patch.object(scraper, "_rotate_ip", new=mock_rotate): + with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"): + await scraper._fetch_serp_html("https://www.avito.ru/test", page=9) + + # 1 transient retry + 1 rotation, оба бюджета тронуты до исчерпания. + assert mock_rotate.await_count == 1 + assert mock_browser.fetch.await_count == 3 -- 2.45.3