fix(avito): post-merge polish — unify fetch_mode guard + empty-render guard (#1811) #1909

Merged
bot-backend merged 1 commit from fix/avito-serp-polish-1811 into main 2026-06-26 07:44:22 +00:00
3 changed files with 17 additions and 2 deletions

View file

@ -757,6 +757,11 @@ async def run_avito_city_sweep(
scraper = AvitoScraper()
if browser_mode:
scraper._browser = shared_bf
# Shared-browser режим: _cffi=None → curl_cffi-fallback на
# firewall пропускается (by design). Логируем для observability.
logger.debug(
"avito pipeline-mode: shared browser only, no cffi fallback"
)
else:
scraper._cffi = session
@ -1324,6 +1329,9 @@ async def run_avito_newbuilding_sweep(
scraper = AvitoScraper()
if browser_mode:
scraper._browser = shared_bf
# Shared-browser режим: _cffi=None → curl_cffi-fallback на firewall
# пропускается (by design). Логируем для observability.
logger.debug("avito pipeline-mode: shared browser only, no cffi fallback")
else:
scraper._cffi = session

View file

@ -352,7 +352,7 @@ class AvitoScraper(BaseScraper):
logger.warning(
"avito: proactive IP rotate at sweep start raised — proceeding", exc_info=True
)
if settings.scraper_fetch_mode != "curl_cffi":
if settings.scraper_fetch_mode == "browser":
# Default (#1805): SERP-фетч через camoufox (tradein-browser per-provider
# + ротирующий backconnect-прокси). curl_cffi поднимаем тут же как
# FALLBACK — если браузер отдаёт firewall/пусто, _fetch_serp_html делает
@ -546,6 +546,12 @@ class AvitoScraper(BaseScraper):
raise AvitoRateLimitedError(
f"avito SERP browser-sidecar error (page={page}): {error_text[:200]}"
) from exc
# Пустой рендер ("" / None) = таймаут рендера sidecar'а, НЕ firewall:
# _is_firewall_page("") вернул бы False → пустой HTML утёк бы как
# «валидный». Возвращаем None — caller (fetch_around) трактует это как
# конец пагинации (break), а не как бан.
if not html:
return None
if not _is_firewall_page(html):
return html
if rot_done < max_rot and await self._rotate_ip():

View file

@ -175,7 +175,8 @@ async def test_fetch_serp_html_browser_mode_returns_html() -> None:
assert result == _NORMAL_HTML
mock_browser.fetch.assert_awaited_once_with("https://www.avito.ru/test")
# _cffi was never set — assert remains None
# _cffi not armed (direct _browser assignment bypasses __aenter__, which would
# otherwise build the curl_cffi fallback session); the browser path never touches it.
assert scraper._cffi is None