feat(tradein): avito SERP via BrowserFetcher behind scraper_fetch_mode (#901) #913
2 changed files with 26 additions and 1 deletions
|
|
@ -221,12 +221,12 @@ class AvitoScraper(BaseScraper):
|
||||||
self.parse_failures: int = 0
|
self.parse_failures: int = 0
|
||||||
|
|
||||||
async def __aenter__(self) -> AvitoScraper:
|
async def __aenter__(self) -> AvitoScraper:
|
||||||
await super().__aenter__()
|
|
||||||
if settings.scraper_fetch_mode == "browser":
|
if settings.scraper_fetch_mode == "browser":
|
||||||
self._browser = BrowserFetcher()
|
self._browser = BrowserFetcher()
|
||||||
await self._browser.__aenter__()
|
await self._browser.__aenter__()
|
||||||
logger.info("avito: SERP fetch via BrowserFetcher (camoufox) — #901")
|
logger.info("avito: SERP fetch via BrowserFetcher (camoufox) — #901")
|
||||||
return self
|
return self
|
||||||
|
await super().__aenter__()
|
||||||
proxies = None
|
proxies = None
|
||||||
if settings.scraper_proxy_url:
|
if settings.scraper_proxy_url:
|
||||||
proxies = {"http": settings.scraper_proxy_url, "https": settings.scraper_proxy_url}
|
proxies = {"http": settings.scraper_proxy_url, "https": settings.scraper_proxy_url}
|
||||||
|
|
@ -286,6 +286,8 @@ class AvitoScraper(BaseScraper):
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
HTML при HTTP 200, либо ``None`` при non-200 (конец пагинации).
|
HTML при HTTP 200, либо ``None`` при non-200 (конец пагинации).
|
||||||
|
В browser-mode HTTP-статус недоступен: метод возвращает HTML или
|
||||||
|
поднимает ``AvitoBlockedError`` (pagination ends via empty-parse in callers).
|
||||||
Raises:
|
Raises:
|
||||||
AvitoBlockedError / AvitoRateLimitedError — если бан не снят ротацией.
|
AvitoBlockedError / AvitoRateLimitedError — если бан не снят ротацией.
|
||||||
"""
|
"""
|
||||||
|
|
|
||||||
|
|
@ -200,3 +200,26 @@ async def test_fetch_city_wide_browser_mode_stops_on_empty_parse() -> None:
|
||||||
assert result[0].source_id == "LOT_1"
|
assert result[0].source_id == "LOT_1"
|
||||||
# page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
|
# page1 parsed (returned lot) + page2 parsed (returned []) → stopped; page3-10 never called
|
||||||
assert call_count == 2
|
assert call_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# 5. rotation returns False → immediate AvitoBlockedError
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_browser_mode_rotation_returns_false_raises_blocked() -> None:
|
||||||
|
"""rotate_url set but _rotate_ip() returns False → must immediately raise AvitoBlockedError.
|
||||||
|
|
||||||
|
Covers the branch: firewall detected + rotate_url configured + _rotate_ip() == False
|
||||||
|
(e.g. rotation API timeout/failure) → we cannot recover → blocked.
|
||||||
|
"""
|
||||||
|
scraper = AvitoScraper()
|
||||||
|
scraper._browser = _make_mock_browser(_FIREWALL_HTML)
|
||||||
|
|
||||||
|
with patch("app.services.scrapers.avito.settings") as mock_settings:
|
||||||
|
mock_settings.avito_proxy_rotate_url = "http://proxy/rotate"
|
||||||
|
mock_settings.avito_proxy_max_rotations = 2
|
||||||
|
with patch.object(scraper, "_rotate_ip", new=AsyncMock(return_value=False)):
|
||||||
|
with pytest.raises(AvitoBlockedError, match="browser-mode"):
|
||||||
|
await scraper._fetch_serp_html("https://avito.ru/test", page=1)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue