diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index 81cddaf6..fbc4b2ee 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -34,6 +34,7 @@ from curl_cffi.requests import AsyncSession from sqlalchemy import text from sqlalchemy.orm import Session +from app.core.config import settings from app.services import scrape_runs from app.services.scrapers.avito import AvitoScraper from app.services.scrapers.avito_detail import fetch_detail, save_detail_enrichment @@ -66,6 +67,19 @@ _CHROME_HEADERS = { } +def _avito_proxies() -> dict[str, str] | None: + """#623: mobile-proxy egress для shared-сессии sweep'а. + + КРИТИЧНО: sweep создаёт собственную shared AsyncSession и присваивает её + `scraper._cffi`, перезатирая проксированную сессию из `AvitoScraper.__aenter__`. + Через эту же сессию идёт и SERP, и detail-энричмент (fetch_detail). Без прокси + detail-страницы летят с datacenter-IP → HTTP 429 → весь run mark_banned. + Пусто (env не задан) → прямое подключение (dev/no-op). + """ + url = settings.avito_proxy_url + return {"http": url, "https": url} if url else None + + # ── Result dataclasses ────────────────────────────────────────── @@ -133,6 +147,7 @@ async def run_avito_pipeline( impersonate="chrome120", timeout=25, headers=_CHROME_HEADERS, + proxies=_avito_proxies(), ) try: @@ -351,6 +366,7 @@ async def run_avito_city_sweep( impersonate="chrome120", timeout=25, headers=_CHROME_HEADERS, + proxies=_avito_proxies(), ) as session: try: for idx, (lat, lon, name) in enumerate(_anchors, start=1): diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py index 7d335cbe..c349a5d7 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py @@ -411,3 +411,22 @@ async def test_fetch_around_blocked_when_rotation_exhausted( await scraper.fetch_around(56.84, 60.60, 1000, pages=1) assert rot.await_count == 1 # одна попытка ротации, потом сдаёмся + + +# ── #623 sweep-pipeline proxy (SERP+detail через mobile proxy) ─── + + +def test_avito_proxies_helper(monkeypatch: pytest.MonkeyPatch) -> None: + """_avito_proxies: None без env, dict {http,https} при заданном AVITO_PROXY_URL. + + Регрессия: sweep создаёт shared-сессию, через неё идут и SERP, и detail — + без прокси detail летит с datacenter-IP → 429 → run banned (#623 followup). + """ + from app.services import scrape_pipeline as sp + + monkeypatch.setattr(sp.settings, "avito_proxy_url", None) + assert sp._avito_proxies() is None + + url = "http://u:p@mproxy.site:16928" + monkeypatch.setattr(sp.settings, "avito_proxy_url", url) + assert sp._avito_proxies() == {"http": url, "https": url}