diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py
index 9f4de0df..9e8ec7f6 100644
--- a/tradein-mvp/backend/app/services/scrapers/avito.py
+++ b/tradein-mvp/backend/app/services/scrapers/avito.py
@@ -27,6 +27,7 @@ import asyncio
import inspect
import logging
import math
+import random
import re
from collections.abc import Callable
from datetime import date, datetime, timedelta, timezone
@@ -272,6 +273,18 @@ _AVITO_429_RECONNECT_RETRIES = 3
_AVITO_403_MAX_RETRIES = 5
_AVITO_403_BACKOFF_SEC = 2.0
+# tradein-browser sidecar (POST /fetch) отдаёт HTTP 500 на ЛЮБУЮ внутреннюю
+# ошибку страницы (тело {"error": "ErrorType: message"}) и HTTP 503 когда браузер
+# не поднимается (прокси упал). Транзиентные сбои (Page.goto Timeout, Target
+# closed, прочие NS_ERROR_/5xx) лечатся коротким backoff-retry того же url БЕЗ
+# ротации IP; soft-ban (NS_ERROR_PROXY_TOO_MANY_REQUESTS / 503 / browser
+# unavailable) — ротацией exit-IP. Когда ОБА бюджета исчерпаны → AvitoRateLimitedError,
+# чтобы run_avito_full_load/run_avito_city_sweep ушли в graceful mark_banned путь
+# (partial preserved), а не в generic except → mark_failed (прод-инцидент run #356:
+# один поздний 500 ронял весь обход после 3186 сохранённых объявлений).
+_AVITO_SIDECAR_TRANSIENT_RETRIES = 2
+_AVITO_SIDECAR_TRANSIENT_BACKOFF_SEC = 2.0
+
# Per-bucket block tolerance в fetch_all_secondary: одиночный 429/блок на ОДНОМ
# (комнатность × ценовой брекет) бакете (напр. deep-pagination dense-бакета) НЕ должен
# ронять весь sweep — бакет скипается, обход продолжается. Только N ПОДРЯД заблокированных
@@ -453,12 +466,86 @@ class AvitoScraper(BaseScraper):
доступен только если _cffi поднят (standalone __aenter__); в
pipeline-режиме с shared-browser _cffi=None → fallback пропускается.
3. curl-fallback тоже firewall/пусто (или недоступен) → AvitoBlockedError.
+
+ Отдельная ветка — sidecar HTTP-ошибка (#356): BrowserFetcher пробрасывает
+ httpx.HTTPStatusError (500/503), если /fetch упал внутри (Page.goto Timeout,
+ NS_ERROR_PROXY_TOO_MANY_REQUESTS, браузер не поднялся). Раньше она всплывала
+ неперехваченной → generic except в run_avito_full_load → mark_failed +
+ re-raise, что роняло весь обход даже после тысяч инкрементально сохранённых
+ карточек. Теперь классифицируем тело ошибки и роутим в ту же
+ ротация+retry-машинерию: soft-ban → ротация IP, transient → bounded backoff,
+ оба бюджета исчерпаны → AvitoRateLimitedError (graceful mark_banned).
"""
assert self._browser is not None
max_rot = settings.avito_proxy_max_rotations if settings.avito_proxy_rotate_url else 0
rot_done = 0
+ # Независимый бюджет коротких retry на transient sidecar-ошибки (timeout/
+ # crash) — НЕ тратит бюджет ротации. Каждая ветка ниже (ротация / transient)
+ # уменьшает свой счётчик → цикл всегда движется к условию исчерпания.
+ transient_left = _AVITO_SIDECAR_TRANSIENT_RETRIES
while True:
- html = await self._browser.fetch(url)
+ try:
+ html = await self._browser.fetch(url)
+ except httpx.HTTPStatusError as exc:
+ # Тело sidecar-ошибки: {"error": "ErrorType: message"}. Парсим
+ # устойчиво — невалидный JSON не должен ронять обработку.
+ status = exc.response.status_code
+ try:
+ error_text = str(exc.response.json().get("error", ""))
+ except Exception:
+ error_text = exc.response.text
+ lowered = error_text.lower()
+
+ is_soft_ban = (
+ "too_many_requests" in lowered
+ or status == 503
+ or "browser unavailable" in lowered
+ or "proxy may be down" in lowered
+ )
+ if is_soft_ban:
+ # Soft proxy rate-limit / прокси недоступен → свежий exit-IP.
+ if rot_done < max_rot and await self._rotate_ip():
+ rot_done += 1
+ logger.info(
+ "avito page=%d sidecar soft-ban (status=%d) — "
+ "retry after rotation #%d: %s",
+ page,
+ status,
+ rot_done,
+ error_text[:200],
+ )
+ continue
+ else:
+ # Transient (Page.goto Timeout / Target closed / прочие 5xx) →
+ # короткий backoff-retry того же url БЕЗ ротации.
+ if transient_left > 0:
+ transient_left -= 1
+ backoff = _AVITO_SIDECAR_TRANSIENT_BACKOFF_SEC + random.uniform(0, 0.5)
+ logger.warning(
+ "avito page=%d sidecar transient (status=%d) — "
+ "retry in %.1fs (left=%d): %s",
+ page,
+ status,
+ backoff,
+ transient_left,
+ error_text[:200],
+ )
+ await asyncio.sleep(backoff)
+ continue
+ # Оба бюджета (ротация + transient) исчерпаны → graceful soft-ban,
+ # чтобы run_avito_full_load/city_sweep ушли в mark_banned (partial
+ # preserved), а НЕ в generic except → mark_failed. Намеренно
+ # AvitoRateLimitedError, не AvitoBlockedError и не bare exception.
+ logger.error(
+ "avito page=%d sidecar error budget exhausted (status=%d) url=%s: %s",
+ page,
+ status,
+ url,
+ error_text[:200],
+ )
+ raise AvitoRateLimitedError(
+ f"avito SERP browser-sidecar error (page={page}): {error_text[:200]}"
+ ) from exc
if not _is_firewall_page(html):
return html
if rot_done < max_rot and await self._rotate_ip():
diff --git a/tradein-mvp/backend/tests/test_avito_browser_mode.py b/tradein-mvp/backend/tests/test_avito_browser_mode.py
index 3959f9fb..95f3eb74 100644
--- a/tradein-mvp/backend/tests/test_avito_browser_mode.py
+++ b/tradein-mvp/backend/tests/test_avito_browser_mode.py
@@ -15,12 +15,13 @@ from __future__ import annotations
import os
from unittest.mock import AsyncMock, MagicMock, patch
+import httpx
import pytest
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
from app.services.scrapers.avito import _FIREWALL_MARKERS, AvitoScraper
-from app.services.scrapers.avito_exceptions import AvitoBlockedError
+from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
from app.services.scrapers.base import ScrapedLot
# ---------------------------------------------------------------------------
@@ -31,6 +32,18 @@ _NORMAL_HTML = "
normal avito SERP"
_FIREWALL_HTML = f"{_FIREWALL_MARKERS[0]}"
+def _sidecar_error(status: int, error: str) -> httpx.HTTPStatusError:
+ """Build a tradein-browser sidecar HTTPStatusError with a real JSON body.
+
+ Mirrors what BrowserFetcher.fetch re-raises: an httpx.HTTPStatusError whose
+ `.response` is intact so `.response.json()["error"]` and `.response.status_code`
+ work the same as in production.
+ """
+ request = httpx.Request("POST", "http://tradein-browser:3000/fetch")
+ response = httpx.Response(status, json={"error": error}, request=request)
+ return httpx.HTTPStatusError(f"{status} error", request=request, response=response)
+
+
def _make_mock_browser(html: str = _NORMAL_HTML) -> MagicMock:
"""Return a mock BrowserFetcher context manager that yields html from fetch()."""
mock = MagicMock()
@@ -358,3 +371,149 @@ async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
with pytest.raises(AvitoBlockedError, match="browser-mode"):
await scraper._fetch_serp_html("https://avito.ru/test", page=1)
+
+
+# ---------------------------------------------------------------------------
+# 6. #356 — sidecar HTTPStatusError (500/503) → graceful AvitoRateLimitedError
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_sidecar_proxy_too_many_requests_rotates_then_succeeds() -> None:
+ """#356: sidecar 500 NS_ERROR_PROXY_TOO_MANY_REQUESTS, rotation available →
+ after IP rotation a retry succeeds and the clean HTML is returned."""
+ scraper = AvitoScraper()
+
+ mock_browser = MagicMock()
+ # First fetch raises the soft-ban 500; after rotation the second fetch is clean.
+ mock_browser.fetch = AsyncMock(
+ side_effect=[
+ _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
+ _NORMAL_HTML,
+ ]
+ )
+ scraper._browser = mock_browser
+ mock_rotate = AsyncMock(return_value=True)
+
+ with patch("app.services.scrapers.avito.settings") as mock_settings:
+ mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
+ mock_settings.avito_proxy_max_rotations = 2
+ with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+ result = await scraper._fetch_serp_html("https://www.avito.ru/test", page=3)
+
+ assert result == _NORMAL_HTML
+ mock_rotate.assert_awaited_once()
+ assert mock_browser.fetch.await_count == 2
+
+
+@pytest.mark.asyncio
+async def test_sidecar_proxy_too_many_requests_exhausted_raises_rate_limited() -> None:
+ """#356: sidecar 500 proxy-too-many with rotation budget exhausted →
+ AvitoRateLimitedError (NOT a bare HTTPStatusError, NOT AvitoBlockedError)."""
+ scraper = AvitoScraper()
+
+ mock_browser = MagicMock()
+ mock_browser.fetch = AsyncMock(
+ side_effect=_sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS")
+ )
+ scraper._browser = mock_browser
+ mock_rotate = AsyncMock(return_value=True)
+
+ with patch("app.services.scrapers.avito.settings") as mock_settings:
+ mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
+ mock_settings.avito_proxy_max_rotations = 2
+ with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+ with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
+ await scraper._fetch_serp_html("https://www.avito.ru/test", page=3)
+
+ # 2 rotations consumed + 1 final fetch that exhausts the budget.
+ assert mock_rotate.await_count == 2
+ assert mock_browser.fetch.await_count == 3
+
+
+@pytest.mark.asyncio
+async def test_sidecar_transient_timeout_exhausts_budget_no_infinite_loop() -> None:
+ """#356: sidecar 500 transient (Timeout) repeated past the transient budget with
+ NO rotations → AvitoRateLimitedError after bounded retries (no infinite loop)."""
+ scraper = AvitoScraper()
+
+ mock_browser = MagicMock()
+ mock_browser.fetch = AsyncMock(
+ side_effect=_sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded")
+ )
+ scraper._browser = mock_browser
+ mock_rotate = AsyncMock(return_value=True)
+
+ with patch("app.services.scrapers.avito.settings") as mock_settings:
+ mock_settings.avito_proxy_rotate_url = None # no rotation → transient budget only
+ mock_settings.avito_proxy_max_rotations = 0
+ # Patch sleep so the bounded backoff doesn't slow the test.
+ with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+ with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+ with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
+ await scraper._fetch_serp_html("https://www.avito.ru/test", page=5)
+
+ # Transient is NOT a soft-ban → never rotates.
+ mock_rotate.assert_not_awaited()
+ # Bounded: _AVITO_SIDECAR_TRANSIENT_RETRIES (2) retries + final attempt = 3.
+ assert mock_browser.fetch.await_count == 3
+
+
+@pytest.mark.asyncio
+async def test_sidecar_503_browser_unavailable_exhausted_raises_rate_limited() -> None:
+ """#356: sidecar 503 'browser unavailable (proxy may be down)' with rotation
+ exhausted → AvitoRateLimitedError (soft-ban classification via status==503)."""
+ scraper = AvitoScraper()
+
+ mock_browser = MagicMock()
+ mock_browser.fetch = AsyncMock(
+ side_effect=_sidecar_error(503, "browser unavailable (proxy may be down)")
+ )
+ scraper._browser = mock_browser
+ mock_rotate = AsyncMock(return_value=True)
+
+ with patch("app.services.scrapers.avito.settings") as mock_settings:
+ mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
+ mock_settings.avito_proxy_max_rotations = 1
+ with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+ with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
+ await scraper._fetch_serp_html("https://www.avito.ru/test", page=7)
+
+ # 1 rotation consumed then exhausted.
+ assert mock_rotate.await_count == 1
+ assert mock_browser.fetch.await_count == 2
+
+
+@pytest.mark.asyncio
+async def test_sidecar_transient_then_soft_ban_independent_budgets() -> None:
+ """#356: смешанная последовательность transient → soft-ban пинит, что бюджеты
+ независимы и корректно чередуются в одном while-цикле.
+
+ attempt 1: transient Timeout → тратит transient_left (1 из 2), backoff, continue.
+ attempt 2: too_many_requests → soft-ban, тратит ротацию (1 из 1), continue.
+ attempt 3: too_many_requests, ротация исчерпана → AvitoRateLimitedError.
+ """
+ scraper = AvitoScraper()
+
+ mock_browser = MagicMock()
+ mock_browser.fetch = AsyncMock(
+ side_effect=[
+ _sidecar_error(500, "TimeoutError: Page.goto: Timeout 60000ms exceeded"),
+ _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
+ _sidecar_error(500, "Error: Page.goto: NS_ERROR_PROXY_TOO_MANY_REQUESTS"),
+ ]
+ )
+ scraper._browser = mock_browser
+ mock_rotate = AsyncMock(return_value=True)
+
+ with patch("app.services.scrapers.avito.settings") as mock_settings:
+ mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
+ mock_settings.avito_proxy_max_rotations = 1
+ with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+ with patch.object(scraper, "_rotate_ip", new=mock_rotate):
+ with pytest.raises(AvitoRateLimitedError, match="browser-sidecar"):
+ await scraper._fetch_serp_html("https://www.avito.ru/test", page=9)
+
+ # 1 transient retry + 1 rotation, оба бюджета тронуты до исчерпания.
+ assert mock_rotate.await_count == 1
+ assert mock_browser.fetch.await_count == 3