"""P2 resilience — page-level retry для avito sweep (citywide / byrooms). Покрывает _fetch_serp_html_with_retry и интеграцию в fetch_city_wide: один транзиентный сбой страницы больше НЕ обрывает весь прогон. Тесты: 1. transient на 1-й попытке → успех на 2-й → возвращает html (1 retry). 2. все попытки кидают generic → re-raise после исчерпания. 3. AvitoBlockedError на 1-й попытке → немедленный re-raise (НЕ retry, call_count==1). 4. None (non-200) → возвращается без retry. 5. integration fetch_city_wide: page=2 транзиентный 1 раз потом ок → обе страницы. 6. integration fetch_city_wide: page=3 persistent-fail → break после retry, страницы 1-2 сохранены. Без сети, без БД, без curl_cffi. """ from __future__ import annotations import os from unittest.mock import AsyncMock, patch import pytest os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") from app.services.scrapers.avito import AvitoScraper from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError from app.services.scrapers.base import ScrapedLot def _make_lot(source_id: str) -> ScrapedLot: return ScrapedLot( source="avito", source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}", source_id=source_id, price_rub=5_000_000, ) # --------------------------------------------------------------------------- # 1. transient → retry → success # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_retry_recovers_after_one_transient() -> None: s = AvitoScraper() calls = 0 async def flaky(url: str, page: int) -> str: nonlocal calls calls += 1 if calls == 1: raise RuntimeError("transient network blip") return "ok" with patch.object(s, "_fetch_serp_html", side_effect=flaky): with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): html = await s._fetch_serp_html_with_retry("u", 1, max_retries=2) assert html == "ok" assert calls == 2 # 1 fail + 1 success # --------------------------------------------------------------------------- # 2. all attempts fail → re-raise # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_retry_reraises_after_exhaustion() -> None: s = AvitoScraper() calls = 0 async def always_fail(url: str, page: int) -> str: nonlocal calls calls += 1 raise RuntimeError("persistent failure") with patch.object(s, "_fetch_serp_html", side_effect=always_fail): with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): with pytest.raises(RuntimeError, match="persistent failure"): await s._fetch_serp_html_with_retry("u", 1, max_retries=2) assert calls == 3 # max_retries=2 → 3 attempts total # --------------------------------------------------------------------------- # 3. hard-block → immediate re-raise, no retry # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_blocked_error_not_retried() -> None: s = AvitoScraper() calls = 0 async def blocked(url: str, page: int) -> str: nonlocal calls calls += 1 raise AvitoBlockedError("IP banned") with patch.object(s, "_fetch_serp_html", side_effect=blocked): with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): with pytest.raises(AvitoBlockedError): await s._fetch_serp_html_with_retry("u", 1, max_retries=2) assert calls == 1 # hard-block: no retry @pytest.mark.asyncio async def test_rate_limited_error_not_retried() -> None: s = AvitoScraper() calls = 0 async def rl(url: str, page: int) -> str: nonlocal calls calls += 1 raise AvitoRateLimitedError("429") with patch.object(s, "_fetch_serp_html", side_effect=rl): with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): with pytest.raises(AvitoRateLimitedError): await s._fetch_serp_html_with_retry("u", 1, max_retries=2) assert calls == 1 # --------------------------------------------------------------------------- # 4. None (non-200) → returned without retry # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_none_returned_without_retry() -> None: s = AvitoScraper() calls = 0 async def returns_none(url: str, page: int) -> None: nonlocal calls calls += 1 return None with patch.object(s, "_fetch_serp_html", side_effect=returns_none): with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): result = await s._fetch_serp_html_with_retry("u", 1, max_retries=2) assert result is None assert calls == 1 # None is end-of-pagination, not an error # --------------------------------------------------------------------------- # 5. integration: transient page recovers, run does NOT abort # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_fetch_city_wide_transient_page_does_not_abort() -> None: """page=2 кидает transient 1 раз, потом ок → собираются обе страницы.""" s = AvitoScraper() page_fail_budget = {2: 1} # page=2 фейлит ровно один раз async def flaky_html(url: str, page: int) -> str: if page_fail_budget.get(page, 0) > 0: page_fail_budget[page] -= 1 raise RuntimeError(f"transient on page {page}") return f"page{page}" page_lots: dict[int, list[ScrapedLot]] = { 1: [_make_lot("P1a"), _make_lot("P1b")], 2: [_make_lot("P2a")], 3: [], # end of pagination } def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]: for p, lots in page_lots.items(): if f"page{p}" in html: return lots return [] with patch.object(s, "_fetch_serp_html", side_effect=flaky_html): with patch.object(s, "_parse_html", side_effect=mock_parse): with patch.object(s, "sleep_between_requests", new=AsyncMock()): with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): result = await s.fetch_city_wide(pages=10, delay_override_sec=0) ids = {lot.source_id for lot in result} assert ids == {"P1a", "P1b", "P2a"} # page=2 survived the transient blip # --------------------------------------------------------------------------- # 6. integration: persistent page failure → break after retry, head kept # --------------------------------------------------------------------------- @pytest.mark.asyncio async def test_fetch_city_wide_persistent_page_breaks_keeping_head() -> None: """page=3 фейлит всегда → break после исчерпания retry; страницы 1-2 сохранены.""" s = AvitoScraper() async def html(url: str, page: int) -> str: if page >= 3: raise RuntimeError(f"persistent on page {page}") return f"page{page}" page_lots: dict[int, list[ScrapedLot]] = { 1: [_make_lot("P1a")], 2: [_make_lot("P2a")], } def mock_parse(html_str: str, source_url_base: str) -> list[ScrapedLot]: for p, lots in page_lots.items(): if f"page{p}" in html_str: return lots return [] with patch.object(s, "_fetch_serp_html", side_effect=html): with patch.object(s, "_parse_html", side_effect=mock_parse): with patch.object(s, "sleep_between_requests", new=AsyncMock()): with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()): result = await s.fetch_city_wide(pages=10, delay_override_sec=0) ids = {lot.source_id for lot in result} assert ids == {"P1a", "P2a"} # pages 1-2 kept; page=3 persistent → graceful break