diff --git a/tradein-mvp/backend/app/services/scrapers/avito_detail.py b/tradein-mvp/backend/app/services/scrapers/avito_detail.py index f7f52ddd..0b2726c1 100644 --- a/tradein-mvp/backend/app/services/scrapers/avito_detail.py +++ b/tradein-mvp/backend/app/services/scrapers/avito_detail.py @@ -19,6 +19,7 @@ from __future__ import annotations +import asyncio import json import logging import re @@ -43,6 +44,15 @@ logger = logging.getLogger(__name__) AVITO_BASE = "https://www.avito.ru" +# Reconnect-retry на detail-странице под backconnect-прокси — зеркало SERP-фикса +# (#1765, avito.py): 403/firewall = залочен лишь текущий exit-IP backconnect-прокси; +# пересоздание curl_cffi-сессии = новый CONNECT-туннель = свежий exit-IP → escape. +# 429 = transient conn-limit, лечится коротким retry на той же сессии (без reconnect). +_AVITO_DETAIL_403_MAX_RETRIES = 5 +_AVITO_DETAIL_403_BACKOFF_SEC = 2.0 +_AVITO_DETAIL_429_MAX_RETRIES = 4 +_AVITO_DETAIL_429_BACKOFF_SEC = 1.5 + # ── Русские месяцы ───────────────────────────────────────────────────────── RUS_MONTHS: dict[str, int] = { "января": 1, @@ -234,6 +244,35 @@ class DetailEnrichment: raw_html_meta: dict[str, Any] = field(default_factory=dict) +def _build_detail_session() -> AsyncSession: + """Создать curl_cffi-сессию для detail-fetch (impersonate chrome120, через прокси + если задан settings.scraper_proxy_url). + + Вынесено из own-session-ветки fetch_detail чтобы reconnect-retry мог пересоздать + эфемерную сессию (новый CONNECT-туннель = свежий backconnect exit-IP) под 403. + proxy=None → прямое подключение. + """ + from app.core.config import settings + + proxy_url = settings.scraper_proxy_url + proxies = {"http": proxy_url, "https": proxy_url} if proxy_url else None + return AsyncSession( + impersonate="chrome120", + timeout=25, + proxies=proxies, + headers={ + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", + "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", + "Cache-Control": "max-age=0", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "Upgrade-Insecure-Requests": "1", + }, + ) + + # ── fetch_detail ────────────────────────────────────────────────────────────── async def fetch_detail( item_url: str, @@ -256,47 +295,81 @@ async def fetch_detail( raise AvitoBlockedError(f"Avito detail firewall (browser-mode) for {full_url}") return parse_detail_html(html, full_url) + from app.core.config import settings + own_session = cffi_session is None if cffi_session is None: # Mobile proxy wiring (#806 follow-up): own-session path (no shared session passed, # e.g. admin endpoint). Caller-provided sessions (scrape_pipeline) already have proxy # from AvitoScraper.__aenter__ — don't override. proxy=None → прямое подключение. - from app.core.config import settings - - _proxy_url = settings.scraper_proxy_url - _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None - session = AsyncSession( - impersonate="chrome120", - timeout=25, - proxies=_proxies, - headers={ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - "Cache-Control": "max-age=0", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "Upgrade-Insecure-Requests": "1", - }, - ) + session = _build_detail_session() else: session = cffi_session full_url = item_url if item_url.startswith("http") else urljoin(AVITO_BASE, item_url) + # curl_cffi egress всегда через scraper_proxy_url (backconnect mproxy). 403/firewall = + # залочен лишь текущий exit-IP → reconnect новой сессией = свежий IP. Гейтим reconnect + # ТОЛЬКО на наличие proxy (зеркало SERP-фикса #1765). Без proxy — старое поведение + # (raise на первом 403). Caller-provided сессию НЕ трогаем: reconnect использует + # эфемерную retry_session, которую сами закрываем в finally. + backconnect = bool(settings.scraper_proxy_url) + r403 = 0 + r429 = 0 + attempt_session = session + retry_session: AsyncSession | None = None try: - response = await session.get(full_url) - if response.status_code == 403: - raise AvitoBlockedError(f"Avito detail HTTP 403 for {full_url}") - if response.status_code == 429: - raise AvitoRateLimitedError(f"Avito detail HTTP 429 for {full_url}") - if response.status_code != 200: - raise ValueError(f"avito detail HTTP {response.status_code} for {full_url}") - enrichment = parse_detail_html(response.text, full_url) - return enrichment + while True: + response = await attempt_session.get(full_url) + sc = response.status_code + is_firewall = sc == 200 and _is_firewall_page(response.text) + + # 429 через backconnect = transient conn-limit, НЕ IP-ban: короткий retry той + # же сессии без reconnect (зеркало SERP). Только при исчерпании — raise. + if sc == 429 and r429 < _AVITO_DETAIL_429_MAX_RETRIES: + r429 += 1 + logger.info( + "avito detail HTTP 429 (backconnect conn-limit) — short retry %d/%d", + r429, + _AVITO_DETAIL_429_MAX_RETRIES, + ) + await asyncio.sleep(_AVITO_DETAIL_429_BACKOFF_SEC) + continue + + if sc == 403 or is_firewall: + if backconnect and r403 < _AVITO_DETAIL_403_MAX_RETRIES: + r403 += 1 + # Эфемерная свежая сессия (новый CONNECT-туннель = свежий exit-IP). + # Caller-provided/own session НЕ пересоздаём — только retry_session. + if retry_session is not None: + try: + await retry_session.close() + except Exception: + pass + retry_session = _build_detail_session() + attempt_session = retry_session + logger.info( + "avito detail 403/firewall %s — backconnect reconnect retry %d/%d", + full_url, + r403, + _AVITO_DETAIL_403_MAX_RETRIES, + ) + await asyncio.sleep(_AVITO_DETAIL_403_BACKOFF_SEC) + continue + raise AvitoBlockedError(f"Avito detail HTTP 403 for {full_url}") + + if sc == 429: + raise AvitoRateLimitedError(f"Avito detail HTTP 429 for {full_url}") + if sc != 200: + raise ValueError(f"avito detail HTTP {sc} for {full_url}") + return parse_detail_html(response.text, full_url) finally: if own_session: await session.close() + if retry_session is not None and retry_session is not session: + try: + await retry_session.close() + except Exception: + pass # ── parse_detail_html ───────────────────────────────────────────────────────── diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_detail_403_reconnect.py b/tradein-mvp/backend/tests/scrapers/test_avito_detail_403_reconnect.py new file mode 100644 index 00000000..3c469222 --- /dev/null +++ b/tradein-mvp/backend/tests/scrapers/test_avito_detail_403_reconnect.py @@ -0,0 +1,191 @@ +"""curl_cffi 403/firewall reconnect-retry на detail-странице под backconnect-прокси. + +fetch_detail ронял весь detail-backfill на первом 403/429 через AvitoBlockedError / +AvitoRateLimitedError (run 191 обогатил 48 → zombie). Зеркало SERP-фикса (#1765): под +backconnect rotating-прокси (mproxy.site) каждое НОВОЕ соединение даёт свежий exit-IP, +поэтому 403/firewall ретраим пересозданием эфемерной curl_cffi-сессии до +_AVITO_DETAIL_403_MAX_RETRIES раз. 429 — короткий retry той же сессии без reconnect. + +Ownership-правило: caller-provided cffi_session НИКОГДА не закрывается/не пересоздаётся — +retries идут через свою эфемерную retry_session (закрываем в finally). + +Включаем backconnect через scraper_proxy_url_env (scraper_proxy_url — read-only property, +его не monkeypatch'ишь). Зеркало fixture-стиля test_avito_403_backconnect_reconnect.py. +""" + +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from app.core.config import settings +from app.services.scrapers import avito_detail as detail_mod +from app.services.scrapers.avito_detail import fetch_detail +from app.services.scrapers.avito_exceptions import AvitoBlockedError + +_SENTINEL = object() +_FIREWALL_HTML = ( + "