fix(tradein/avito): route sidecar timeout/transport errors as transient, not crash

avito_full_load has been failing since 2026-07-02 (8 runs status=failed, some with error=''). Root cause: _fetch_serp_html_browser only caught httpx.HTTPStatusError from BrowserFetcher.fetch. When the sidecar fetch fails with httpx.TimeoutException/httpx.TransportError (ReadTimeout/ConnectError/ RemoteProtocolError) after BrowserFetcher's own internal retry, the exception was unhandled and propagated through _paginate_incremental_bracket into run_avito_full_load's generic `except Exception` -> mark_failed(str(exc)). str(ReadTimeout())=="" produced an empty error, and mark_failed does not persist done_buckets (unlike mark_banned), losing the resume checkpoint. Widen the except clause to also catch httpx.TimeoutException/TransportError, mirroring the existing pattern in _probe_total. Non-HTTPStatusError exceptions have no HTTP status, so they're classified status=0 with a descriptive error_text and routed through the existing transient-backoff branch (not soft-ban — status=0/exception text never matches the soft-ban keyword checks). Once the bounded retry budget is exhausted, they raise AvitoRateLimitedError, same as a sidecar HTTPStatusError -> run_avito_full_load's dedicated except (AvitoBlockedError, AvitoRateLimitedError) -> mark_banned, which DOES persist done_buckets, giving graceful partial-save instead of a crash. Does NOT fix the underlying cause (mobile proxy instability) -- that's infra/devops, out of scope. This only makes the failure mode graceful.
2026-07-13 23:52:48 +03:00 · 2026-07-13 23:52:48 +03:00 · 6573edba51
commit 6573edba51
parent c904fbf94e
2 changed files with 75 additions and 15 deletions
--- a/tradein-mvp/backend/tests/test_pipeline_browser_routing.py
+++ b/tradein-mvp/backend/tests/test_pipeline_browser_routing.py
@ -278,3 +278,45 @@ async def test_fetch_serp_html_routes_through_browser() -> None:

    assert html == "<html><body>real serp listing</body></html>"
    scraper._browser.fetch.assert_awaited_once()
+
+
+# ── 6. SERP browser sidecar timeout → transient retry, not bare exception ────
+# #2472: BrowserFetcher.fetch (browser_fetcher.py) retries once internally on
+# httpx.HTTPError/TransportError, then re-raises. Before this fix, non-
+# HTTPStatusError exceptions (httpx.TimeoutException/TransportError — e.g.
+# ReadTimeout/ConnectError/RemoteProtocolError) propagated unhandled through
+# _fetch_serp_html_browser straight into pipeline.run_avito_full_load's generic
+# `except Exception` → mark_failed(str(exc)) (str(ReadTimeout())=="" → empty
+# error, and mark_failed does NOT persist done_buckets, unlike mark_banned).
+
+
+@pytest.mark.asyncio
+async def test_fetch_serp_html_browser_timeout_routes_transient_not_bare_exception() -> None:
+    """httpx.ReadTimeout from browser.fetch() must NOT propagate as an unhandled
+    exception. It's classified transient (status=0, no soft-ban keyword match)
+    and, once the bounded retry budget is exhausted, surfaces as
+    AvitoRateLimitedError — the same graceful banned-path a sidecar
+    HTTPStatusError takes — never as a bare/generic exception."""
+    from unittest.mock import patch
+
+    import httpx
+    from scraper_kit.avito_exceptions import AvitoRateLimitedError
+    from scraper_kit.providers.avito import serp as serp_module
+    from scraper_kit.providers.avito.serp import AvitoScraper
+
+    from app.services.scraper_adapters import RealScraperConfig
+
+    scraper = AvitoScraper(RealScraperConfig())
+    assert scraper._cffi is None
+    scraper._browser = AsyncMock()
+    scraper._browser.fetch = AsyncMock(side_effect=httpx.ReadTimeout(""))
+
+    with patch.object(serp_module.asyncio, "sleep", AsyncMock()):
+        with pytest.raises(AvitoRateLimitedError):
+            await scraper._fetch_serp_html(
+                "https://www.avito.ru/ekaterinburg/kvartiry/prodam", page=1
+            )
+
+    # 1 initial attempt + 2 transient retries (_AVITO_SIDECAR_TRANSIENT_RETRIES) = 3,
+    # no IP rotation involved (RealScraperConfig has no rotate_url configured in tests).
+    assert scraper._browser.fetch.await_count == 3
--- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/avito/serp.py
+++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/avito/serp.py
@ -485,14 +485,19 @@ class AvitoScraper(BaseScraper):
             pipeline-режиме с shared-browser _cffi=None → fallback пропускается.
          3. curl-fallback тоже firewall/пусто (или недоступен) → AvitoBlockedError.

-        Отдельная ветка — sidecar HTTP-ошибка (#356): BrowserFetcher пробрасывает
+        Отдельная ветка — sidecar-ошибка (#356, #2472): BrowserFetcher пробрасывает
        httpx.HTTPStatusError (500/503), если /fetch упал внутри (Page.goto Timeout,
-        NS_ERROR_PROXY_TOO_MANY_REQUESTS, браузер не поднялся). Раньше она всплывала
-        неперехваченной → generic except в run_avito_full_load → mark_failed +
-        re-raise, что роняло весь обход даже после тысяч инкрементально сохранённых
-        карточек. Теперь классифицируем тело ошибки и роутим в ту же
-        ротация+retry-машинерию: soft-ban → ротация IP, transient → bounded backoff,
-        оба бюджета исчерпаны → AvitoRateLimitedError (graceful mark_banned).
+        NS_ERROR_PROXY_TOO_MANY_REQUESTS, браузер не поднялся), либо
+        httpx.TimeoutException/httpx.TransportError (ReadTimeout/ConnectError/
+        RemoteProtocolError), если соединение до sidecar оборвалось ДО ответа (после
+        internal-retry в BrowserFetcher.fetch). Раньше обе ветки (а до #2472 — только
+        не-HTTPStatusError) всплывали неперехваченными → generic except в
+        run_avito_full_load → mark_failed + re-raise, что роняло весь обход даже после
+        тысяч инкрементально сохранённых карточек (и теряло done_buckets-чекпоинт —
+        mark_failed его не пишет). Теперь классифицируем тело ошибки (HTTP-ошибки) или
+        просто помечаем как transient (не-HTTP) и роутим в ту же ротация+retry-
+        машинерию: soft-ban → ротация IP, transient → bounded backoff, оба бюджета
+        исчерпаны → AvitoRateLimitedError (graceful mark_banned, partial preserved).
        """
        assert self._browser is not None
        max_rot = self._config.avito_proxy_max_rotations if self._config.avito_proxy_rotate_url else 0
@ -504,14 +509,27 @@ class AvitoScraper(BaseScraper):
        while True:
            try:
                html = await self._browser.fetch(url)
-            except httpx.HTTPStatusError as exc:
-                # Тело sidecar-ошибки: {"error": "ErrorType: message"}. Парсим
-                # устойчиво — невалидный JSON не должен ронять обработку.
+            except (httpx.HTTPStatusError, httpx.TimeoutException, httpx.TransportError) as exc:
+                # HTTPStatusError: тело sidecar-ошибки {"error": "ErrorType: message"}.
+                # Парсим устойчиво — невалидный JSON не должен ронять обработку.
+                #
+                # TimeoutException/TransportError (ReadTimeout/ConnectError/
+                # RemoteProtocolError): сеть до sidecar оборвалась ДО ответа (после
+                # internal-retry BrowserFetcher.fetch, browser_fetcher.py) — HTTP-статуса
+                # нет. Раньше это вываливалось необработанным через
+                # _paginate_incremental_bracket в generic except run_avito_full_load →
+                # mark_failed(str(exc)) — а str(ReadTimeout())=="" (пустой error) теряет
+                # done_buckets чекпоинт (mark_failed его не пишет, в отличие от mark_banned).
+                # Роутим как transient (status=0, не soft-ban) — симметрично _probe_total.
+                if isinstance(exc, httpx.HTTPStatusError):
                    status = exc.response.status_code
                    try:
                        error_text = str(exc.response.json().get("error", ""))
                    except Exception:
                        error_text = exc.response.text
+                else:
+                    status = 0
+                    error_text = f"{type(exc).__name__}: {exc}"
                lowered = error_text.lower()

                is_soft_ban = (