Merge pull request 'fix(tradein): avito sweep page-level retry — transient page error no longer aborts run' (#1755) from fix/avito-sweep-page-retry into main

tradein-mvp/backend/app/services/scrapers/avito.py

@@ -431,6 +431,45 @@ class AvitoScraper(BaseScraper):
                 return None
             return response.text
 
+    async def _fetch_serp_html_with_retry(
+        self, url: str, page: int, *, max_retries: int = 2
+    ) -> str | None:
+        """Page-level retry поверх _fetch_serp_html для sweep-обходов (P2 resilience).
+
+        _fetch_serp_html уже ретраит транзиентные fetch-обрывы и ротирует IP, но
+        если после внутренних попыток он всё равно поднимает generic Exception —
+        внешний sweep-цикл (citywide/byrooms) делал break и терял весь хвост.
+        Здесь даём странице ещё ``max_retries`` попыток перед тем как пробросить.
+
+        Поведение:
+          - AvitoBlockedError / AvitoRateLimitedError — hard-block, пробрасываются
+            немедленно (НЕ retry).
+          - generic Exception — если попытки остались: warning + короткий backoff +
+            следующая попытка; иначе re-raise последнего исключения (внешний цикл
+            решит break).
+          - None — это «non-200 / конец пагинации», НЕ ошибка → возвращается как есть
+            без ретраев.
+        """
+        attempts = max_retries + 1
+        backoff = max(2.0, min(self.request_delay_sec, 4.0))
+        for attempt in range(1, attempts + 1):
+            try:
+                return await self._fetch_serp_html(url, page)
+            except (AvitoBlockedError, AvitoRateLimitedError):
+                raise
+            except Exception:
+                if attempt >= attempts:
+                    raise
+                logger.warning(
+                    "avito page=%d transient error — retry (attempt %d/%d)",
+                    page,
+                    attempt,
+                    max_retries,
+                    exc_info=True,
+                )
+                await asyncio.sleep(backoff)
+        return None  # pragma: no cover — loop either returns or raises
+
     # ── Public ──────────────────────────────────────────────────────────────
     async def fetch_around(
         self,
@@ -964,11 +1003,13 @@ class AvitoScraper(BaseScraper):
         for page in range(1, pages + 1):
             url = url_builder(page)
             try:
-                html = await self._fetch_serp_html(url, page)
+                html = await self._fetch_serp_html_with_retry(url, page)
             except (AvitoBlockedError, AvitoRateLimitedError):
                 raise
             except Exception:
-                logger.exception("avito %s page=%d fetch failed url=%s", label, page, url)
+                logger.exception(
+                    "avito %s page=%d fetch failed after retries url=%s", label, page, url
+                )
                 break
             if html is None:
                 logger.info("avito %s page=%d: non-200 — end of pagination", label, page)
@@ -1069,12 +1110,12 @@ class AvitoScraper(BaseScraper):
             for page in range(1, pages + 1):
                 url = self._build_rooms_url(slug, page)
                 try:
-                    html = await self._fetch_serp_html(url, page)
+                    html = await self._fetch_serp_html_with_retry(url, page)
                 except (AvitoBlockedError, AvitoRateLimitedError):
                     raise
                 except Exception:
                     logger.exception(
-                        "avito byrooms category=%s page=%d fetch failed url=%s",
+                        "avito byrooms category=%s page=%d fetch failed after retries url=%s",
                         name,
                         page,
                         url,

tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py

@@ -0,0 +1,225 @@
+"""P2 resilience — page-level retry для avito sweep (citywide / byrooms).
+
+Покрывает _fetch_serp_html_with_retry и интеграцию в fetch_city_wide:
+один транзиентный сбой страницы больше НЕ обрывает весь прогон.
+
+Тесты:
+1. transient на 1-й попытке → успех на 2-й → возвращает html (1 retry).
+2. все попытки кидают generic → re-raise после исчерпания.
+3. AvitoBlockedError на 1-й попытке → немедленный re-raise (НЕ retry, call_count==1).
+4. None (non-200) → возвращается без retry.
+5. integration fetch_city_wide: page=2 транзиентный 1 раз потом ок → обе страницы.
+6. integration fetch_city_wide: page=3 persistent-fail → break после retry,
+   страницы 1-2 сохранены.
+
+Без сети, без БД, без curl_cffi.
+"""
+
+from __future__ import annotations
+
+import os
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
+
+from app.services.scrapers.avito import AvitoScraper
+from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
+from app.services.scrapers.base import ScrapedLot
+
+
+def _make_lot(source_id: str) -> ScrapedLot:
+    return ScrapedLot(
+        source="avito",
+        source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}",
+        source_id=source_id,
+        price_rub=5_000_000,
+    )
+
+
+# ---------------------------------------------------------------------------
+# 1. transient → retry → success
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_retry_recovers_after_one_transient() -> None:
+    s = AvitoScraper()
+    calls = 0
+
+    async def flaky(url: str, page: int) -> str:
+        nonlocal calls
+        calls += 1
+        if calls == 1:
+            raise RuntimeError("transient network blip")
+        return "<html>ok</html>"
+
+    with patch.object(s, "_fetch_serp_html", side_effect=flaky):
+        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+            html = await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
+
+    assert html == "<html>ok</html>"
+    assert calls == 2  # 1 fail + 1 success
+
+
+# ---------------------------------------------------------------------------
+# 2. all attempts fail → re-raise
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_retry_reraises_after_exhaustion() -> None:
+    s = AvitoScraper()
+    calls = 0
+
+    async def always_fail(url: str, page: int) -> str:
+        nonlocal calls
+        calls += 1
+        raise RuntimeError("persistent failure")
+
+    with patch.object(s, "_fetch_serp_html", side_effect=always_fail):
+        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+            with pytest.raises(RuntimeError, match="persistent failure"):
+                await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
+
+    assert calls == 3  # max_retries=2 → 3 attempts total
+
+
+# ---------------------------------------------------------------------------
+# 3. hard-block → immediate re-raise, no retry
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_blocked_error_not_retried() -> None:
+    s = AvitoScraper()
+    calls = 0
+
+    async def blocked(url: str, page: int) -> str:
+        nonlocal calls
+        calls += 1
+        raise AvitoBlockedError("IP banned")
+
+    with patch.object(s, "_fetch_serp_html", side_effect=blocked):
+        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+            with pytest.raises(AvitoBlockedError):
+                await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
+
+    assert calls == 1  # hard-block: no retry
+
+
+@pytest.mark.asyncio
+async def test_rate_limited_error_not_retried() -> None:
+    s = AvitoScraper()
+    calls = 0
+
+    async def rl(url: str, page: int) -> str:
+        nonlocal calls
+        calls += 1
+        raise AvitoRateLimitedError("429")
+
+    with patch.object(s, "_fetch_serp_html", side_effect=rl):
+        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+            with pytest.raises(AvitoRateLimitedError):
+                await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
+
+    assert calls == 1
+
+
+# ---------------------------------------------------------------------------
+# 4. None (non-200) → returned without retry
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_none_returned_without_retry() -> None:
+    s = AvitoScraper()
+    calls = 0
+
+    async def returns_none(url: str, page: int) -> None:
+        nonlocal calls
+        calls += 1
+        return None
+
+    with patch.object(s, "_fetch_serp_html", side_effect=returns_none):
+        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+            result = await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
+
+    assert result is None
+    assert calls == 1  # None is end-of-pagination, not an error
+
+
+# ---------------------------------------------------------------------------
+# 5. integration: transient page recovers, run does NOT abort
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_fetch_city_wide_transient_page_does_not_abort() -> None:
+    """page=2 кидает transient 1 раз, потом ок → собираются обе страницы."""
+    s = AvitoScraper()
+    page_fail_budget = {2: 1}  # page=2 фейлит ровно один раз
+
+    async def flaky_html(url: str, page: int) -> str:
+        if page_fail_budget.get(page, 0) > 0:
+            page_fail_budget[page] -= 1
+            raise RuntimeError(f"transient on page {page}")
+        return f"<html>page{page}</html>"
+
+    page_lots: dict[int, list[ScrapedLot]] = {
+        1: [_make_lot("P1a"), _make_lot("P1b")],
+        2: [_make_lot("P2a")],
+        3: [],  # end of pagination
+    }
+
+    def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
+        for p, lots in page_lots.items():
+            if f"page{p}" in html:
+                return lots
+        return []
+
+    with patch.object(s, "_fetch_serp_html", side_effect=flaky_html):
+        with patch.object(s, "_parse_html", side_effect=mock_parse):
+            with patch.object(s, "sleep_between_requests", new=AsyncMock()):
+                with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+                    result = await s.fetch_city_wide(pages=10, delay_override_sec=0)
+
+    ids = {lot.source_id for lot in result}
+    assert ids == {"P1a", "P1b", "P2a"}  # page=2 survived the transient blip
+
+
+# ---------------------------------------------------------------------------
+# 6. integration: persistent page failure → break after retry, head kept
+# ---------------------------------------------------------------------------
+
+
+@pytest.mark.asyncio
+async def test_fetch_city_wide_persistent_page_breaks_keeping_head() -> None:
+    """page=3 фейлит всегда → break после исчерпания retry; страницы 1-2 сохранены."""
+    s = AvitoScraper()
+
+    async def html(url: str, page: int) -> str:
+        if page >= 3:
+            raise RuntimeError(f"persistent on page {page}")
+        return f"<html>page{page}</html>"
+
+    page_lots: dict[int, list[ScrapedLot]] = {
+        1: [_make_lot("P1a")],
+        2: [_make_lot("P2a")],
+    }
+
+    def mock_parse(html_str: str, source_url_base: str) -> list[ScrapedLot]:
+        for p, lots in page_lots.items():
+            if f"page{p}" in html_str:
+                return lots
+        return []
+
+    with patch.object(s, "_fetch_serp_html", side_effect=html):
+        with patch.object(s, "_parse_html", side_effect=mock_parse):
+            with patch.object(s, "sleep_between_requests", new=AsyncMock()):
+                with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
+                    result = await s.fetch_city_wide(pages=10, delay_override_sec=0)
+
+    ids = {lot.source_id for lot in result}
+    assert ids == {"P1a", "P2a"}  # pages 1-2 kept; page=3 persistent → graceful break