feat(tradein): avito SERP via BrowserFetcher behind scraper_fetch_mode (#901) #913

Merged
bot-reviewer merged 2 commits from feat/901-avito-browser-mode into main 2026-05-31 16:49:36 +00:00
2 changed files with 26 additions and 1 deletions
Showing only changes of commit 80b953d0bc - Show all commits

View file

@ -221,12 +221,12 @@ class AvitoScraper(BaseScraper):
self.parse_failures: int = 0 self.parse_failures: int = 0
async def __aenter__(self) -> AvitoScraper: async def __aenter__(self) -> AvitoScraper:
await super().__aenter__()
if settings.scraper_fetch_mode == "browser": if settings.scraper_fetch_mode == "browser":
self._browser = BrowserFetcher() self._browser = BrowserFetcher()
await self._browser.__aenter__() await self._browser.__aenter__()
logger.info("avito: SERP fetch via BrowserFetcher (camoufox) — #901") logger.info("avito: SERP fetch via BrowserFetcher (camoufox) — #901")
return self return self
await super().__aenter__()
proxies = None proxies = None
if settings.scraper_proxy_url: if settings.scraper_proxy_url:
proxies = {"http": settings.scraper_proxy_url, "https": settings.scraper_proxy_url} proxies = {"http": settings.scraper_proxy_url, "https": settings.scraper_proxy_url}
@ -286,6 +286,8 @@ class AvitoScraper(BaseScraper):
Returns: Returns:
HTML при HTTP 200, либо ``None`` при non-200 (конец пагинации). HTML при HTTP 200, либо ``None`` при non-200 (конец пагинации).
В browser-mode HTTP-статус недоступен: метод возвращает HTML или
поднимает ``AvitoBlockedError`` (pagination ends via empty-parse in callers).
Raises: Raises:
AvitoBlockedError / AvitoRateLimitedError если бан не снят ротацией. AvitoBlockedError / AvitoRateLimitedError если бан не снят ротацией.
""" """

View file

@ -200,3 +200,26 @@ async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
assert result[0].source_id == "LOT_1" assert result[0].source_id == "LOT_1"
# page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called # page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
assert call_count == 2 assert call_count == 2
# ---------------------------------------------------------------------------
# 5. rotation returns False → immediate AvitoBlockedError
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
"""rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.
Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
(e.g. rotation API timeout/failure) we cannot recover blocked.
"""
scraper = AvitoScraper()
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
with patch("app.services.scrapers.avito.settings") as mock_settings:
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
mock_settings.avito_proxy_max_rotations = 2
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
with pytest.raises(AvitoBlockedError, match="browser-mode"):
await scraper._fetch_serp_html("https://avito.ru/test", page=1)