fix(avito): graceful sidecar HTTP 500/503 in browser SERP fetch (#1790)

Classify tradein-browser sidecar HTTP 500/503 in _fetch_serp_html_browser and route into existing rotation/retry machinery: soft-ban → IP rotation, transient → bounded backoff, both exhausted → AvitoRateLimitedError (graceful mark_banned, partial preserved) instead of mark_failed. Fixes run-#356 class where one late 500 nuked a run that had saved 3186 listings.

Refs #1790

tradein-mvp/backend/app/services/scrapers/avito.py

@@ -27,6 +27,7 @@ import asyncio
 import inspect
 import logging
 import math
+import random
 import re
 from collections.abc import Callable
 from datetime import date, datetime, timedelta, timezone
@@ -272,6 +273,18 @@ _AVITO_429_RECONNECT_RETRIES = 3
 _AVITO_403_MAX_RETRIES = 5
 _AVITO_403_BACKOFF_SEC = 2.0
 
+# tradein-browser sidecar (POST /fetch) отдаёт HTTP 500 на ЛЮБУЮ внутреннюю
+# ошибку страницы (тело {"error": "ErrorType: message"}) и HTTP 503 когда браузер
+# не поднимается (прокси упал). Транзиентные сбои (Page.goto Timeout, Target
+# closed, прочие NS_ERROR_/5xx) лечатся коротким backoff-retry того же url БЕЗ
+# ротации IP; soft-ban (NS_ERROR_PROXY_TOO_MANY_REQUESTS / 503 / browser
+# unavailable) — ротацией exit-IP. Когда ОБА бюджета исчерпаны → AvitoRateLimitedError,
+# чтобы run_avito_full_load/run_avito_city_sweep ушли в graceful mark_banned путь
+# (partial preserved), а не в generic except → mark_failed (прод-инцидент run #356:
+# один поздний 500 ронял весь обход после 3186 сохранённых объявлений).
+_AVITO_SIDECAR_TRANSIENT_RETRIES = 2
+_AVITO_SIDECAR_TRANSIENT_BACKOFF_SEC = 2.0
+
 # Per-bucket block tolerance в fetch_all_secondary: одиночный 429/блок на ОДНОМ
 # (комнатность × ценовой брекет) бакете (напр. deep-pagination dense-бакета) НЕ должен
 # ронять весь sweep — бакет скипается, обход продолжается. Только N ПОДРЯД заблокированных
@@ -453,12 +466,86 @@ class AvitoScraper(BaseScraper):
              доступен только если _cffi поднят (standalone __aenter__); в
              pipeline-режиме с shared-browser _cffi=None → fallback пропускается.
           3. curl-fallback тоже firewall/пусто (или недоступен) → AvitoBlockedError.
+
+        Отдельная ветка — sidecar HTTP-ошибка (#356): BrowserFetcher пробрасывает
+        httpx.HTTPStatusError (500/503), если /fetch упал внутри (Page.goto Timeout,
+        NS_ERROR_PROXY_TOO_MANY_REQUESTS, браузер не поднялся). Раньше она всплывала
+        неперехваченной → generic except в run_avito_full_load → mark_failed +
+        re-raise, что роняло весь обход даже после тысяч инкрементально сохранённых
+        карточек. Теперь классифицируем тело ошибки и роутим в ту же
+        ротация+retry-машинерию: soft-ban → ротация IP, transient → bounded backoff,
+        оба бюджета исчерпаны → AvitoRateLimitedError (graceful mark_banned).
         """
         assert self._browser is not None
         max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0
         rot_done = 0
+        # Независимый бюджет коротких retry на transient sidecar-ошибки (timeout/
+        # crash) — НЕ тратит бюджет ротации. Каждая ветка ниже (ротация / transient)
+        # уменьшает свой счётчик → цикл всегда движется к условию исчерпания.
+        transient_left = _AVITO_SIDECAR_TRANSIENT_RETRIES
         while True:
-            html = await self._browser.fetch(url)
+            try:
+                html = await self._browser.fetch(url)
+            except httpx.HTTPStatusError as exc:
+                # Тело sidecar-ошибки: {"error": "ErrorType: message"}. Парсим
+                # устойчиво — невалидный JSON не должен ронять обработку.
+                status = exc.response.status_code
+                try:
+                    error_text = str(exc.response.json().get("error", ""))
+                except Exception:
+                    error_text = exc.response.text
+                lowered = error_text.lower()
+
+                is_soft_ban = (
+                    "too_many_requests" in lowered
+                    or status == 503
+                    or "browser unavailable" in lowered
+                    or "proxy may be down" in lowered
+                )
+                if is_soft_ban:
+                    # Soft proxy rate-limit / прокси недоступен → свежий exit-IP.
+                    if rot_done < max_rot and await self._rotate_ip():
+                        rot_done += 1
+                        logger.info(
+                            "avito page=%d sidecar soft-ban (status=%d) — "
+                            "retry after rotation #%d: %s",
+                            page,
+                            status,
+                            rot_done,
+                            error_text[:200],
+                        )
+                        continue
+                else:
+                    # Transient (Page.goto Timeout / Target closed / прочие 5xx) →
+                    # короткий backoff-retry того же url БЕЗ ротации.
+                    if transient_left > 0:
+                        transient_left -= 1
+                        backoff = _AVITO_SIDECAR_TRANSIENT_BACKOFF_SEC + random.uniform(0, 0.5)
+                        logger.warning(
+                            "avito page=%d sidecar transient (status=%d) — "
+                            "retry in %.1fs (left=%d): %s",
+                            page,
+                            status,
+                            backoff,
+                            transient_left,
+                            error_text[:200],
+                        )
+                        await asyncio.sleep(backoff)
+                        continue
+                # Оба бюджета (ротация + transient) исчерпаны → graceful soft-ban,
+                # чтобы run_avito_full_load/city_sweep ушли в mark_banned (partial
+                # preserved), а НЕ в generic except → mark_failed. Намеренно
+                # AvitoRateLimitedError, не AvitoBlockedError и не bare exception.
+                logger.error(
+                    "avito page=%d sidecar error budget exhausted (status=%d) url=%s: %s",
+                    page,
+                    status,
+                    url,
+                    error_text[:200],
+                )
+                raise AvitoRateLimitedError(
+                    f"avito SERP browser-sidecar error (page={page}): {error_text[:200]}"
+                ) from exc
             if not _is_firewall_page(html):
                 return html
             if rot_done < max_rot and await self._rotate_ip():

tradein-mvp/backend/tests/test_avito_browser_mode.py

@@ -15,12 +15,13 @@ from __future__ import annotations
 import os
 from unittest.mock import AsyncMock, MagicMock, patch
 
+import httpx
 import pytest
 
 os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
 
 from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper
-from app.services.scrapers.avito_exceptions import AvitoBlockedError
+from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
 from app.services.scrapers.base import ScrapedLot
 
 # ---------------------------------------------------------------------------
@@ -31,6 +32,18 @@ _NORMAL_HTML = "<html><body>normal avito SERP</body></html>"
 _FIREWALL_HTML = f"<html><head><title>{_FIREWALL_MARKERS[0]}</title></head></html>"
 
 
+def _sidecar_error(status: int, error: str) -> httpx.HTTPStatusError:
+    """Build a tradein-browser sidecar HTTPStatusError with a real JSON body.
+
+    Mirrors what BrowserFetcher.fetch re-raises: an httpx.HTTPStatusError whose
+    `.response` is intact so `.response.json()["error"]` and `.response.status_code`
+    work the same as in production.
+    """
+    request = httpx.Request("POST", "http://tradein-browser:3000/fetch")
+    response = httpx.Response(status, json={"error": error}, request=request)
+    return httpx.HTTPStatusError(f"{status} error", request=request, response=response)
+
+
 def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock:
     """Return a mock BrowserFetcher context manager that yields html from fetch()."""
     mock = MagicMock()
@@ -358,3 +371,149 @@ async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
         with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
             with pytest.raises(AvitoBlockedError, match="browser-mode"):
                 await scraper._fetch_serp_html("https://avito.ru/test", page=1)
+
+
+# ---------------------------------------------------------------------------
+# 6. #356 — sidecar HTTPStatusError (500/503) → graceful AvitoRateLimitedError
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_sidecar_proxy_too_many_requests_rotates_then_succeeds() -> None:
+    """#356: sidecar 500 NS_ERROR_PROXY_TOO_MANY_REQUESTS, rotation available →
+    after IP rotation a retry succeeds and the clean HTML is returned."""
+    scraper = AvitoScraper()
+
+    mock_browser = MagicMock()
+    # First fetch raises the soft-ban 500; after rotation the second fetch is clean.
+    mock_browser.fetch = AsyncMock(
+        side_effect=[
+            _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
+            _NORMAL_HTML,
+        ]
+    )
+    scraper._browser = mock_browser
+    mock_rotate = AsyncMock(return_value=True)
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
+        mock_settings.avito_proxy_max_rotations = 2
+        with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+            result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=3)
+
+    assert result == _NORMAL_HTML
+    mock_rotate.assert_awaited_once()
+    assert mock_browser.fetch.await_count == 2
+
+
+@pytest.mark.asyncio
+async def test_sidecar_proxy_too_many_requests_exhausted_raises_rate_limited() -> None:
+    """#356: sidecar 500 proxy-too-many with rotation budget exhausted →
+    AvitoRateLimitedError (NOT a bare HTTPStatusError, NOT AvitoBlockedError)."""
+    scraper = AvitoScraper()
+
+    mock_browser = MagicMock()
+    mock_browser.fetch = AsyncMock(
+        side_effect=_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS")
+    )
+    scraper._browser = mock_browser
+    mock_rotate = AsyncMock(return_value=True)
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
+        mock_settings.avito_proxy_max_rotations = 2
+        with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+            with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
+                await scraper._fetch_serp_html("https://www.avito.ru/test", page=3)
+
+    # 2 rotations consumed + 1 final fetch that exhausts the budget.
+    assert mock_rotate.await_count == 2
+    assert mock_browser.fetch.await_count == 3
+
+
+@pytest.mark.asyncio
+async def test_sidecar_transient_timeout_exhausts_budget_no_infinite_loop() -> None:
+    """#356: sidecar 500 transient (Timeout) repeated past the transient budget with
+    NO rotations → AvitoRateLimitedError after bounded retries (no infinite loop)."""
+    scraper = AvitoScraper()
+
+    mock_browser = MagicMock()
+    mock_browser.fetch = AsyncMock(
+        side_effect=_sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded")
+    )
+    scraper._browser = mock_browser
+    mock_rotate = AsyncMock(return_value=True)
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.avito_proxy_rotate_url = None  # no rotation → transient budget only
+        mock_settings.avito_proxy_max_rotations = 0
+        # Patch sleep so the bounded backoff doesn't slow the test.
+        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+            with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+                with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
+                    await scraper._fetch_serp_html("https://www.avito.ru/test", page=5)
+
+    # Transient is NOT a soft-ban → never rotates.
+    mock_rotate.assert_not_awaited()
+    # Bounded: _AVITO_SIDECAR_TRANSIENT_RETRIES (2) retries + final attempt = 3.
+    assert mock_browser.fetch.await_count == 3
+
+
+@pytest.mark.asyncio
+async def test_sidecar_503_browser_unavailable_exhausted_raises_rate_limited() -> None:
+    """#356: sidecar 503 'browser unavailable (proxy may be down)' with rotation
+    exhausted → AvitoRateLimitedError (soft-ban classification via status==503)."""
+    scraper = AvitoScraper()
+
+    mock_browser = MagicMock()
+    mock_browser.fetch = AsyncMock(
+        side_effect=_sidecar_error(503, "browser unavailable (proxy may be down)")
+    )
+    scraper._browser = mock_browser
+    mock_rotate = AsyncMock(return_value=True)
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
+        mock_settings.avito_proxy_max_rotations = 1
+        with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+            with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
+                await scraper._fetch_serp_html("https://www.avito.ru/test", page=7)
+
+    # 1 rotation consumed then exhausted.
+    assert mock_rotate.await_count == 1
+    assert mock_browser.fetch.await_count == 2
+
+
+@pytest.mark.asyncio
+async def test_sidecar_transient_then_soft_ban_independent_budgets() -> None:
+    """#356: смешанная последовательность transient → soft-ban пинит, что бюджеты
+    независимы и корректно чередуются в одном while-цикле.
+
+    attempt 1: transient Timeout → тратит transient_left (1 из 2), backoff, continue.
+    attempt 2: too_many_requests → soft-ban, тратит ротацию (1 из 1), continue.
+    attempt 3: too_many_requests, ротация исчерпана → AvitoRateLimitedError.
+    """
+    scraper = AvitoScraper()
+
+    mock_browser = MagicMock()
+    mock_browser.fetch = AsyncMock(
+        side_effect=[
+            _sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded"),
+            _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
+            _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
+        ]
+    )
+    scraper._browser = mock_browser
+    mock_rotate = AsyncMock(return_value=True)
+
+    with patch("app.services.scrapers.avito.settings") as mock_settings:
+        mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
+        mock_settings.avito_proxy_max_rotations = 1
+        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+            with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+                with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
+                    await scraper._fetch_serp_html("https://www.avito.ru/test", page=9)
+
+    # 1 transient retry + 1 rotation, оба бюджета тронуты до исчерпания.
+    assert mock_rotate.await_count == 1
+    assert mock_browser.fetch.await_count == 3