tradein-mvp/backend/app/services/scrapers/avito.py

@@ -431,45 +431,6 @@ class AvitoScraper(BaseScraper):
                 return None
             return response.text
 
-    async def _fetch_serp_html_with_retry(
-        self, url: str, page: int, *, max_retries: int = 2
-    ) -> str | None:
-        """Page-level retry поверх _fetch_serp_html для sweep-обходов (P2 resilience).
-
-        _fetch_serp_html уже ретраит транзиентные fetch-обрывы и ротирует IP, но
-        если после внутренних попыток он всё равно поднимает generic Exception —
-        внешний sweep-цикл (citywide/byrooms) делал break и терял весь хвост.
-        Здесь даём странице ещё ``max_retries`` попыток перед тем как пробросить.
-
-        Поведение:
-          - AvitoBlockedError / AvitoRateLimitedError — hard-block, пробрасываются
-            немедленно (НЕ retry).
-          - generic Exception — если попытки остались: warning + короткий backoff +
-            следующая попытка; иначе re-raise последнего исключения (внешний цикл
-            решит break).
-          - None — это «non-200 / конец пагинации», НЕ ошибка → возвращается как есть
-            без ретраев.
-        """
-        attempts = max_retries + 1
-        backoff = max(2.0, min(self.request_delay_sec, 4.0))
-        for attempt in range(1, attempts + 1):
-            try:
-                return await self._fetch_serp_html(url, page)
-            except (AvitoBlockedError, AvitoRateLimitedError):
-                raise
-            except Exception:
-                if attempt >= attempts:
-                    raise
-                logger.warning(
-                    "avito page=%d transient error — retry (attempt %d/%d)",
-                    page,
-                    attempt,
-                    max_retries,
-                    exc_info=True,
-                )
-                await asyncio.sleep(backoff)
-        return None  # pragma: no cover — loop either returns or raises
-
     # ── Public ──────────────────────────────────────────────────────────────
     async def fetch_around(
         self,
@@ -1003,13 +964,11 @@ class AvitoScraper(BaseScraper):
         for page in range(1, pages + 1):
             url = url_builder(page)
             try:
-                html = await self._fetch_serp_html_with_retry(url, page)
+                html = await self._fetch_serp_html(url, page)
             except (AvitoBlockedError, AvitoRateLimitedError):
                 raise
             except Exception:
-                logger.exception(
+                logger.exception("avito %s page=%d fetch failed url=%s", label, page, url)
-                    "avito %s page=%d fetch failed after retries url=%s", label, page, url
-                )
                 break
             if html is None:
                 logger.info("avito %s page=%d: non-200 — end of pagination", label, page)
@@ -1110,12 +1069,12 @@ class AvitoScraper(BaseScraper):
             for page in range(1, pages + 1):
                 url = self._build_rooms_url(slug, page)
                 try:
-                    html = await self._fetch_serp_html_with_retry(url, page)
+                    html = await self._fetch_serp_html(url, page)
                 except (AvitoBlockedError, AvitoRateLimitedError):
                     raise
                 except Exception:
                     logger.exception(
-                        "avito byrooms category=%s page=%d fetch failed after retries url=%s",
+                        "avito byrooms category=%s page=%d fetch failed url=%s",
                         name,
                         page,
                         url,

tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py

@@ -1,225 +0,0 @@
-"""P2 resilience — page-level retry для avito sweep (citywide / byrooms).
-
-Покрывает _fetch_serp_html_with_retry и интеграцию в fetch_city_wide:
-один транзиентный сбой страницы больше НЕ обрывает весь прогон.
-
-Тесты:
-1. transient на 1-й попытке → успех на 2-й → возвращает html (1 retry).
-2. все попытки кидают generic → re-raise после исчерпания.
-3. AvitoBlockedError на 1-й попытке → немедленный re-raise (НЕ retry, call_count==1).
-4. None (non-200) → возвращается без retry.
-5. integration fetch_city_wide: page=2 транзиентный 1 раз потом ок → обе страницы.
-6. integration fetch_city_wide: page=3 persistent-fail → break после retry,
-   страницы 1-2 сохранены.
-
-Без сети, без БД, без curl_cffi.
-"""
-
-from __future__ import annotations
-
-import os
-from unittest.mock import AsyncMock, patch
-
-import pytest
-
-os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
-
-from app.services.scrapers.avito import AvitoScraper
-from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
-from app.services.scrapers.base import ScrapedLot
-
-
-def _make_lot(source_id: str) -> ScrapedLot:
-    return ScrapedLot(
-        source="avito",
-        source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}",
-        source_id=source_id,
-        price_rub=5_000_000,
-    )
-
-
-# ---------------------------------------------------------------------------
-# 1. transient → retry → success
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_retry_recovers_after_one_transient() -> None:
-    s = AvitoScraper()
-    calls = 0
-
-    async def flaky(url: str, page: int) -> str:
-        nonlocal calls
-        calls += 1
-        if calls == 1:
-            raise RuntimeError("transient network blip")
-        return "<html>ok</html>"
-
-    with patch.object(s, "_fetch_serp_html", side_effect=flaky):
-        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
-            html = await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
-
-    assert html == "<html>ok</html>"
-    assert calls == 2  # 1 fail + 1 success
-
-
-# ---------------------------------------------------------------------------
-# 2. all attempts fail → re-raise
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_retry_reraises_after_exhaustion() -> None:
-    s = AvitoScraper()
-    calls = 0
-
-    async def always_fail(url: str, page: int) -> str:
-        nonlocal calls
-        calls += 1
-        raise RuntimeError("persistent failure")
-
-    with patch.object(s, "_fetch_serp_html", side_effect=always_fail):
-        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
-            with pytest.raises(RuntimeError, match="persistent failure"):
-                await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
-
-    assert calls == 3  # max_retries=2 → 3 attempts total
-
-
-# ---------------------------------------------------------------------------
-# 3. hard-block → immediate re-raise, no retry
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_blocked_error_not_retried() -> None:
-    s = AvitoScraper()
-    calls = 0
-
-    async def blocked(url: str, page: int) -> str:
-        nonlocal calls
-        calls += 1
-        raise AvitoBlockedError("IP banned")
-
-    with patch.object(s, "_fetch_serp_html", side_effect=blocked):
-        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
-            with pytest.raises(AvitoBlockedError):
-                await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
-
-    assert calls == 1  # hard-block: no retry
-
-
-@pytest.mark.asyncio
-async def test_rate_limited_error_not_retried() -> None:
-    s = AvitoScraper()
-    calls = 0
-
-    async def rl(url: str, page: int) -> str:
-        nonlocal calls
-        calls += 1
-        raise AvitoRateLimitedError("429")
-
-    with patch.object(s, "_fetch_serp_html", side_effect=rl):
-        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
-            with pytest.raises(AvitoRateLimitedError):
-                await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
-
-    assert calls == 1
-
-
-# ---------------------------------------------------------------------------
-# 4. None (non-200) → returned without retry
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_none_returned_without_retry() -> None:
-    s = AvitoScraper()
-    calls = 0
-
-    async def returns_none(url: str, page: int) -> None:
-        nonlocal calls
-        calls += 1
-        return None
-
-    with patch.object(s, "_fetch_serp_html", side_effect=returns_none):
-        with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
-            result = await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
-
-    assert result is None
-    assert calls == 1  # None is end-of-pagination, not an error
-
-
-# ---------------------------------------------------------------------------
-# 5. integration: transient page recovers, run does NOT abort
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_fetch_city_wide_transient_page_does_not_abort() -> None:
-    """page=2 кидает transient 1 раз, потом ок → собираются обе страницы."""
-    s = AvitoScraper()
-    page_fail_budget = {2: 1}  # page=2 фейлит ровно один раз
-
-    async def flaky_html(url: str, page: int) -> str:
-        if page_fail_budget.get(page, 0) > 0:
-            page_fail_budget[page] -= 1
-            raise RuntimeError(f"transient on page {page}")
-        return f"<html>page{page}</html>"
-
-    page_lots: dict[int, list[ScrapedLot]] = {
-        1: [_make_lot("P1a"), _make_lot("P1b")],
-        2: [_make_lot("P2a")],
-        3: [],  # end of pagination
-    }
-
-    def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
-        for p, lots in page_lots.items():
-            if f"page{p}" in html:
-                return lots
-        return []
-
-    with patch.object(s, "_fetch_serp_html", side_effect=flaky_html):
-        with patch.object(s, "_parse_html", side_effect=mock_parse):
-            with patch.object(s, "sleep_between_requests", new=AsyncMock()):
-                with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
-                    result = await s.fetch_city_wide(pages=10, delay_override_sec=0)
-
-    ids = {lot.source_id for lot in result}
-    assert ids == {"P1a", "P1b", "P2a"}  # page=2 survived the transient blip
-
-
-# ---------------------------------------------------------------------------
-# 6. integration: persistent page failure → break after retry, head kept
-# ---------------------------------------------------------------------------
-
-
-@pytest.mark.asyncio
-async def test_fetch_city_wide_persistent_page_breaks_keeping_head() -> None:
-    """page=3 фейлит всегда → break после исчерпания retry; страницы 1-2 сохранены."""
-    s = AvitoScraper()
-
-    async def html(url: str, page: int) -> str:
-        if page >= 3:
-            raise RuntimeError(f"persistent on page {page}")
-        return f"<html>page{page}</html>"
-
-    page_lots: dict[int, list[ScrapedLot]] = {
-        1: [_make_lot("P1a")],
-        2: [_make_lot("P2a")],
-    }
-
-    def mock_parse(html_str: str, source_url_base: str) -> list[ScrapedLot]:
-        for p, lots in page_lots.items():
-            if f"page{p}" in html_str:
-                return lots
-        return []
-
-    with patch.object(s, "_fetch_serp_html", side_effect=html):
-        with patch.object(s, "_parse_html", side_effect=mock_parse):
-            with patch.object(s, "sleep_between_requests", new=AsyncMock()):
-                with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
-                    result = await s.fetch_city_wide(pages=10, delay_override_sec=0)
-
-    ids = {lot.source_id for lot in result}
-    assert ids == {"P1a", "P2a"}  # pages 1-2 kept; page=3 persistent → graceful break