225 lines
8.3 KiB
Python
225 lines
8.3 KiB
Python
"""P2 resilience — page-level retry для avito sweep (citywide / byrooms).
|
||
|
||
Покрывает _fetch_serp_html_with_retry и интеграцию в fetch_city_wide:
|
||
один транзиентный сбой страницы больше НЕ обрывает весь прогон.
|
||
|
||
Тесты:
|
||
1. transient на 1-й попытке → успех на 2-й → возвращает html (1 retry).
|
||
2. все попытки кидают generic → re-raise после исчерпания.
|
||
3. AvitoBlockedError на 1-й попытке → немедленный re-raise (НЕ retry, call_count==1).
|
||
4. None (non-200) → возвращается без retry.
|
||
5. integration fetch_city_wide: page=2 транзиентный 1 раз потом ок → обе страницы.
|
||
6. integration fetch_city_wide: page=3 persistent-fail → break после retry,
|
||
страницы 1-2 сохранены.
|
||
|
||
Без сети, без БД, без curl_cffi.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import os
|
||
from unittest.mock import AsyncMock, patch
|
||
|
||
import pytest
|
||
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
from app.services.scrapers.avito import AvitoScraper
|
||
from app.services.scrapers.avito_exceptions import AvitoBlockedError, AvitoRateLimitedError
|
||
from app.services.scrapers.base import ScrapedLot
|
||
|
||
|
||
def _make_lot(source_id: str) -> ScrapedLot:
|
||
return ScrapedLot(
|
||
source="avito",
|
||
source_url=f"https://www.avito.ru/ekaterinburg/kvartiry/{source_id}",
|
||
source_id=source_id,
|
||
price_rub=5_000_000,
|
||
)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 1. transient → retry → success
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_retry_recovers_after_one_transient() -> None:
|
||
s = AvitoScraper()
|
||
calls = 0
|
||
|
||
async def flaky(url: str, page: int) -> str:
|
||
nonlocal calls
|
||
calls += 1
|
||
if calls == 1:
|
||
raise RuntimeError("transient network blip")
|
||
return "<html>ok</html>"
|
||
|
||
with patch.object(s, "_fetch_serp_html", side_effect=flaky):
|
||
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
|
||
html = await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
|
||
|
||
assert html == "<html>ok</html>"
|
||
assert calls == 2 # 1 fail + 1 success
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 2. all attempts fail → re-raise
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_retry_reraises_after_exhaustion() -> None:
|
||
s = AvitoScraper()
|
||
calls = 0
|
||
|
||
async def always_fail(url: str, page: int) -> str:
|
||
nonlocal calls
|
||
calls += 1
|
||
raise RuntimeError("persistent failure")
|
||
|
||
with patch.object(s, "_fetch_serp_html", side_effect=always_fail):
|
||
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
|
||
with pytest.raises(RuntimeError, match="persistent failure"):
|
||
await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
|
||
|
||
assert calls == 3 # max_retries=2 → 3 attempts total
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 3. hard-block → immediate re-raise, no retry
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_blocked_error_not_retried() -> None:
|
||
s = AvitoScraper()
|
||
calls = 0
|
||
|
||
async def blocked(url: str, page: int) -> str:
|
||
nonlocal calls
|
||
calls += 1
|
||
raise AvitoBlockedError("IP banned")
|
||
|
||
with patch.object(s, "_fetch_serp_html", side_effect=blocked):
|
||
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
|
||
with pytest.raises(AvitoBlockedError):
|
||
await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
|
||
|
||
assert calls == 1 # hard-block: no retry
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_rate_limited_error_not_retried() -> None:
|
||
s = AvitoScraper()
|
||
calls = 0
|
||
|
||
async def rl(url: str, page: int) -> str:
|
||
nonlocal calls
|
||
calls += 1
|
||
raise AvitoRateLimitedError("429")
|
||
|
||
with patch.object(s, "_fetch_serp_html", side_effect=rl):
|
||
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
|
||
with pytest.raises(AvitoRateLimitedError):
|
||
await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
|
||
|
||
assert calls == 1
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 4. None (non-200) → returned without retry
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_none_returned_without_retry() -> None:
|
||
s = AvitoScraper()
|
||
calls = 0
|
||
|
||
async def returns_none(url: str, page: int) -> None:
|
||
nonlocal calls
|
||
calls += 1
|
||
return None
|
||
|
||
with patch.object(s, "_fetch_serp_html", side_effect=returns_none):
|
||
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
|
||
result = await s._fetch_serp_html_with_retry("u", 1, max_retries=2)
|
||
|
||
assert result is None
|
||
assert calls == 1 # None is end-of-pagination, not an error
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 5. integration: transient page recovers, run does NOT abort
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_city_wide_transient_page_does_not_abort() -> None:
|
||
"""page=2 кидает transient 1 раз, потом ок → собираются обе страницы."""
|
||
s = AvitoScraper()
|
||
page_fail_budget = {2: 1} # page=2 фейлит ровно один раз
|
||
|
||
async def flaky_html(url: str, page: int) -> str:
|
||
if page_fail_budget.get(page, 0) > 0:
|
||
page_fail_budget[page] -= 1
|
||
raise RuntimeError(f"transient on page {page}")
|
||
return f"<html>page{page}</html>"
|
||
|
||
page_lots: dict[int, list[ScrapedLot]] = {
|
||
1: [_make_lot("P1a"), _make_lot("P1b")],
|
||
2: [_make_lot("P2a")],
|
||
3: [], # end of pagination
|
||
}
|
||
|
||
def mock_parse(html: str, source_url_base: str) -> list[ScrapedLot]:
|
||
for p, lots in page_lots.items():
|
||
if f"page{p}" in html:
|
||
return lots
|
||
return []
|
||
|
||
with patch.object(s, "_fetch_serp_html", side_effect=flaky_html):
|
||
with patch.object(s, "_parse_html", side_effect=mock_parse):
|
||
with patch.object(s, "sleep_between_requests", new=AsyncMock()):
|
||
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
|
||
result = await s.fetch_city_wide(pages=10, delay_override_sec=0)
|
||
|
||
ids = {lot.source_id for lot in result}
|
||
assert ids == {"P1a", "P1b", "P2a"} # page=2 survived the transient blip
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# 6. integration: persistent page failure → break after retry, head kept
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_fetch_city_wide_persistent_page_breaks_keeping_head() -> None:
|
||
"""page=3 фейлит всегда → break после исчерпания retry; страницы 1-2 сохранены."""
|
||
s = AvitoScraper()
|
||
|
||
async def html(url: str, page: int) -> str:
|
||
if page >= 3:
|
||
raise RuntimeError(f"persistent on page {page}")
|
||
return f"<html>page{page}</html>"
|
||
|
||
page_lots: dict[int, list[ScrapedLot]] = {
|
||
1: [_make_lot("P1a")],
|
||
2: [_make_lot("P2a")],
|
||
}
|
||
|
||
def mock_parse(html_str: str, source_url_base: str) -> list[ScrapedLot]:
|
||
for p, lots in page_lots.items():
|
||
if f"page{p}" in html_str:
|
||
return lots
|
||
return []
|
||
|
||
with patch.object(s, "_fetch_serp_html", side_effect=html):
|
||
with patch.object(s, "_parse_html", side_effect=mock_parse):
|
||
with patch.object(s, "sleep_between_requests", new=AsyncMock()):
|
||
with patch("app.services.scrapers.avito.asyncio.sleep", new=AsyncMock()):
|
||
result = await s.fetch_city_wide(pages=10, delay_override_sec=0)
|
||
|
||
ids = {lot.source_id for lot in result}
|
||
assert ids == {"P1a", "P2a"} # pages 1-2 kept; page=3 persistent → graceful break
|