feat(tradein): avito SERP via BrowserFetcher behind scraper_fetch_mode (#901) #913
2 changed files with 26 additions and 1 deletions
|
|
@ -221,12 +221,12 @@ class AvitoScraper(BaseScraper):
|
|||
self.parse_failures: int = 0
|
||||
|
||||
async def __aenter__(self) -> AvitoScraper:
|
||||
await super().__aenter__()
|
||||
if settings.scraper_fetch_mode == "browser":
|
||||
self._browser = BrowserFetcher()
|
||||
await self._browser.__aenter__()
|
||||
logger.info("avito: SERP fetch via BrowserFetcher (camoufox) — #901")
|
||||
return self
|
||||
await super().__aenter__()
|
||||
proxies = None
|
||||
if settings.scraper_proxy_url:
|
||||
proxies = {"http": settings.scraper_proxy_url, "https": settings.scraper_proxy_url}
|
||||
|
|
@ -286,6 +286,8 @@ class AvitoScraper(BaseScraper):
|
|||
|
||||
Returns:
|
||||
HTML при HTTP 200, либо ``None`` при non-200 (конец пагинации).
|
||||
В browser-mode HTTP-статус недоступен: метод возвращает HTML или
|
||||
поднимает ``AvitoBlockedError`` (pagination ends via empty-parse in callers).
|
||||
Raises:
|
||||
AvitoBlockedError / AvitoRateLimitedError — если бан не снят ротацией.
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -200,3 +200,26 @@ async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
|
|||
assert result[0].source_id == "LOT_1"
|
||||
# page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
|
||||
assert call_count == 2
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 5. rotation returns False → immediate AvitoBlockedError
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
|
||||
"""rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.
|
||||
|
||||
Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
|
||||
(e.g. rotation API timeout/failure) → we cannot recover → blocked.
|
||||
"""
|
||||
scraper = AvitoScraper()
|
||||
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
|
||||
|
||||
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||||
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
|
||||
mock_settings.avito_proxy_max_rotations = 2
|
||||
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
|
||||
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
||||
await scraper._fetch_serp_html("https://avito.ru/test", page=1)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue