From 6c132d7f0ae1d3f165c03bacfea0263c28ba55d7 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Thu, 18 Jun 2026 14:29:01 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein):=20avito=20sweep=20page-level=20re?= =?UTF-8?q?try=20=E2=80=94=20transient=20page=20error=20no=20longer=20abor?= =?UTF-8?q?ts=20run?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/app/services/scrapers/avito.py | 49 +++- .../tests/scrapers/test_avito_sweep_retry.py | 225 ++++++++++++++++++ 2 files changed, 270 insertions(+), 4 deletions(-) create mode 100644 tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py diff --git a/tradein-mvp/backend/app/services/scrapers/avito.py b/tradein-mvp/backend/app/services/scrapers/avito.py index 3f081fd8..ab5667a1 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito.py +++ b/tradein-mvp/backend/app/services/scrapers/avito.py @@ -431,6 +431,45 @@ class AvitoScraper(BaseScraper): return None return response.text + async def _fetch_serp_html_with_retry( + self, url: str, page: int, *, max_retries: int = 2 + ) -> str | None: + """Page-level retry поверх _fetch_serp_html для sweep-обходов (P2 resilience). + + _fetch_serp_html уже ретраит транзиентные fetch-обрывы и ротирует IP, но + если после внутренних попыток он всё равно поднимает generic Exception — + внешний sweep-цикл (citywide/byrooms) делал break и терял весь хвост. + Здесь даём странице ещё ``max_retries`` попыток перед тем как пробросить. + + Поведение: + - AvitoBlockedError / AvitoRateLimitedError — hard-block, пробрасываются + немедленно (НЕ retry). + - generic Exception — если попытки остались: warning + короткий backoff + + следующая попытка; иначе re-raise последнего исключения (внешний цикл + решит break). + - None — это «non-200 / конец пагинации», НЕ ошибка → возвращается как есть + без ретраев. + """ + attempts = max_retries + 1 + backoff = max(2.0, min(self.request_delay_sec, 4.0)) + for attempt in range(1, attempts + 1): + try: + return await self._fetch_serp_html(url, page) + except (AvitoBlockedError, AvitoRateLimitedError): + raise + except Exception: + if attempt >= attempts: + raise + logger.warning( + "avito page=%d transient error — retry (attempt %d/%d)", + page, + attempt, + max_retries, + exc_info=True, + ) + await asyncio.sleep(backoff) + return None # pragma: no cover — loop either returns or raises + # ── Public ────────────────────────────────────────────────────────────── async def fetch_around( self, @@ -964,11 +1003,13 @@ class AvitoScraper(BaseScraper): for page in range(1, pages + 1): url = url_builder(page) try: - html = await self._fetch_serp_html(url, page) + html = await self._fetch_serp_html_with_retry(url, page) except (AvitoBlockedError, AvitoRateLimitedError): raise except Exception: - logger.exception("avito %s page=%d fetch failed url=%s", label, page, url) + logger.exception( + "avito %s page=%d fetch failed after retries url=%s", label, page, url + ) break if html is None: logger.info("avito %s page=%d: non-200 — end of pagination", label, page) @@ -1069,12 +1110,12 @@ class AvitoScraper(BaseScraper): for page in range(1, pages + 1): url = self._build_rooms_url(slug, page) try: - html = await self._fetch_serp_html(url, page) + html = await self._fetch_serp_html_with_retry(url, page) except (AvitoBlockedError, AvitoRateLimitedError): raise except Exception: logger.exception( - "avito byrooms category=%s page=%d fetch failed url=%s", + "avito byrooms category=%s page=%d fetch failed after retries url=%s", name, page, url, diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py b/tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py new file mode 100644 index 00000000..269eca06 --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_avito_sweep_retry.py @@ -0,0 +1,225 @@ +"""P2 resilience — page-level retry для avito sweep (citywide / byrooms). + +Покрывает _fetch_serp_html_with_retry и интеграцию в fetch_city_wide: +один транзиентный сбой страницы больше НЕ обрывает весь прогон. + +Тесты: +1. transient на 1-й попытке → успех на 2-й → возвращает html (1 retry). +2. все попытки кидают generic → re-raise после исчерпания. +3. AvitoBlockedError на 1-й попытке → немедленный re-raise (НЕ retry, call_count==1). +4. None (non-200) → возвращается без retry. +5. integration fetch_city_wide: page=2 транзиентный 1 раз потом ок → обе страницы. +6. integration fetch_city_wide: page=3 persistent-fail → break после retry, + страницы 1-2 сохранены. + +Без сети, без БД, без curl_cffi. +""" + +from __future__ import annotations + +import os +from unittest.mock import AsyncMock, patch + +import pytest + +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +from app.services.scrapers.avito import AvitoScraper +from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError +from app.services.scrapers.base import ScrapedLot + + +def _make_lot(source_id: str) -> ScrapedLot: + return ScrapedLot( + source="avito", + source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}", + source_id=source_id, + price_rub=5_000_000, + ) + + +# --------------------------------------------------------------------------- +# 1. transient → retry → success +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_retry_recovers_after_one_transient() -> None: + s = AvitoScraper() + calls = 0 + + async def flaky(url: str, page: int) -> str: + nonlocal calls + calls += 1 + if calls == 1: + raise RuntimeError("transient network blip") + return "ok" + + with patch.object(s, "_fetch_serp_html", side_effect=flaky): + with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): + html = await s._fetch_serp_html_with_retry("u", 1, max_retries=2) + + assert html == "ok" + assert calls == 2 # 1 fail + 1 success + + +# --------------------------------------------------------------------------- +# 2. all attempts fail → re-raise +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_retry_reraises_after_exhaustion() -> None: + s = AvitoScraper() + calls = 0 + + async def always_fail(url: str, page: int) -> str: + nonlocal calls + calls += 1 + raise RuntimeError("persistent failure") + + with patch.object(s, "_fetch_serp_html", side_effect=always_fail): + with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): + with pytest.raises(RuntimeError, match="persistent failure"): + await s._fetch_serp_html_with_retry("u", 1, max_retries=2) + + assert calls == 3 # max_retries=2 → 3 attempts total + + +# --------------------------------------------------------------------------- +# 3. hard-block → immediate re-raise, no retry +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_blocked_error_not_retried() -> None: + s = AvitoScraper() + calls = 0 + + async def blocked(url: str, page: int) -> str: + nonlocal calls + calls += 1 + raise AvitoBlockedError("IP banned") + + with patch.object(s, "_fetch_serp_html", side_effect=blocked): + with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): + with pytest.raises(AvitoBlockedError): + await s._fetch_serp_html_with_retry("u", 1, max_retries=2) + + assert calls == 1 # hard-block: no retry + + +@pytest.mark.asyncio +async def test_rate_limited_error_not_retried() -> None: + s = AvitoScraper() + calls = 0 + + async def rl(url: str, page: int) -> str: + nonlocal calls + calls += 1 + raise AvitoRateLimitedError("429") + + with patch.object(s, "_fetch_serp_html", side_effect=rl): + with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): + with pytest.raises(AvitoRateLimitedError): + await s._fetch_serp_html_with_retry("u", 1, max_retries=2) + + assert calls == 1 + + +# --------------------------------------------------------------------------- +# 4. None (non-200) → returned without retry +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_none_returned_without_retry() -> None: + s = AvitoScraper() + calls = 0 + + async def returns_none(url: str, page: int) -> None: + nonlocal calls + calls += 1 + return None + + with patch.object(s, "_fetch_serp_html", side_effect=returns_none): + with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): + result = await s._fetch_serp_html_with_retry("u", 1, max_retries=2) + + assert result is None + assert calls == 1 # None is end-of-pagination, not an error + + +# --------------------------------------------------------------------------- +# 5. integration: transient page recovers, run does NOT abort +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_fetch_city_wide_transient_page_does_not_abort() -> None: + """page=2 кидает transient 1 раз, потом ок → собираются обе страницы.""" + s = AvitoScraper() + page_fail_budget = {2: 1} # page=2 фейлит ровно один раз + + async def flaky_html(url: str, page: int) -> str: + if page_fail_budget.get(page, 0) > 0: + page_fail_budget[page] -= 1 + raise RuntimeError(f"transient on page {page}") + return f"page{page}" + + page_lots: dict[int, list[ScrapedLot]] = { + 1: [_make_lot("P1a"), _make_lot("P1b")], + 2: [_make_lot("P2a")], + 3: [], # end of pagination + } + + def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: + for p, lots in page_lots.items(): + if f"page{p}" in html: + return lots + return [] + + with patch.object(s, "_fetch_serp_html", side_effect=flaky_html): + with patch.object(s, "_parse_html", side_effect=mock_parse): + with patch.object(s, "sleep_between_requests", new=AsyncMock()): + with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): + result = await s.fetch_city_wide(pages=10, delay_override_sec=0) + + ids = {lot.source_id for lot in result} + assert ids == {"P1a", "P1b", "P2a"} # page=2 survived the transient blip + + +# --------------------------------------------------------------------------- +# 6. integration: persistent page failure → break after retry, head kept +# --------------------------------------------------------------------------- + + +@pytest.mark.asyncio +async def test_fetch_city_wide_persistent_page_breaks_keeping_head() -> None: + """page=3 фейлит всегда → break после исчерпания retry; страницы 1-2 сохранены.""" + s = AvitoScraper() + + async def html(url: str, page: int) -> str: + if page >= 3: + raise RuntimeError(f"persistent on page {page}") + return f"page{page}" + + page_lots: dict[int, list[ScrapedLot]] = { + 1: [_make_lot("P1a")], + 2: [_make_lot("P2a")], + } + + def mock_parse(html_str: str, source_url_base: str) -> list[ScrapedLot]: + for p, lots in page_lots.items(): + if f"page{p}" in html_str: + return lots + return [] + + with patch.object(s, "_fetch_serp_html", side_effect=html): + with patch.object(s, "_parse_html", side_effect=mock_parse): + with patch.object(s, "sleep_between_requests", new=AsyncMock()): + with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): + result = await s.fetch_city_wide(pages=10, delay_override_sec=0) + + ids = {lot.source_id for lot in result} + assert ids == {"P1a", "P2a"} # pages 1-2 kept; page=3 persistent → graceful break -- 2.45.3