From 950d0f60de28312581766d404825c6c7118cdc37 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 30 May 2026 18:07:37 +0000 Subject: [PATCH] =?UTF-8?q?fix(tradein):=20sweep=20shared-session=20(SERP+?= =?UTF-8?q?detail)=20=D1=87=D0=B5=D1=80=D0=B5=D0=B7=20mobile=20proxy=20(#6?= =?UTF-8?q?23)=20(#805)=20Co-authored-by:=20bot-backend=20=20Co-committed-by:=20bot-backend=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/app/services/scrape_pipeline.py | 16 ++++++++++++++++ .../tests/scrapers/test_avito_anti_bot.py | 19 +++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/tradein-mvp/backend/app/services/scrape_pipeline.py b/tradein-mvp/backend/app/services/scrape_pipeline.py index 81cddaf6..fbc4b2ee 100644 --- a/tradein-mvp/backend/app/services/scrape_pipeline.py +++ b/tradein-mvp/backend/app/services/scrape_pipeline.py @@ -34,6 +34,7 @@ from curl_cffi.requests import AsyncSession from sqlalchemy import text from sqlalchemy.orm import Session +from app.core.config import settings from app.services import scrape_runs from app.services.scrapers.avito import AvitoScraper from app.services.scrapers.avito_detail import fetch_detail, save_detail_enrichment @@ -66,6 +67,19 @@ _CHROME_HEADERS = { } +def _avito_proxies() -> dict[str, str] | None: + """#623: mobile-proxy egress для shared-сессии sweep'а. + + КРИТИЧНО: sweep создаёт собственную shared AsyncSession и присваивает её + `scraper._cffi`, перезатирая проксированную сессию из `AvitoScraper.__aenter__`. + Через эту же сессию идёт и SERP, и detail-энричмент (fetch_detail). Без прокси + detail-страницы летят с datacenter-IP → HTTP 429 → весь run mark_banned. + Пусто (env не задан) → прямое подключение (dev/no-op). + """ + url = settings.avito_proxy_url + return {"http": url, "https": url} if url else None + + # ── Result dataclasses ────────────────────────────────────────── @@ -133,6 +147,7 @@ async def run_avito_pipeline( impersonate="chrome120", timeout=25, headers=_CHROME_HEADERS, + proxies=_avito_proxies(), ) try: @@ -351,6 +366,7 @@ async def run_avito_city_sweep( impersonate="chrome120", timeout=25, headers=_CHROME_HEADERS, + proxies=_avito_proxies(), ) as session: try: for idx, (lat, lon, name) in enumerate(_anchors, start=1): diff --git a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py index 7d335cbe..c349a5d7 100644 --- a/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py +++ b/tradein-mvp/backend/tests/scrapers/test_avito_anti_bot.py @@ -411,3 +411,22 @@ async def test_fetch_around_blocked_when_rotation_exhausted( await scraper.fetch_around(56.84, 60.60, 1000, pages=1) assert rot.await_count == 1 # одна попытка ротации, потом сдаёмся + + +# ── #623 sweep-pipeline proxy (SERP+detail через mobile proxy) ─── + + +def test_avito_proxies_helper(monkeypatch: pytest.MonkeyPatch) -> None: + """_avito_proxies: None без env, dict {http,https} при заданном AVITO_PROXY_URL. + + Регрессия: sweep создаёт shared-сессию, через неё идут и SERP, и detail — + без прокси detail летит с datacenter-IP → 429 → run banned (#623 followup). + """ + from app.services import scrape_pipeline as sp + + monkeypatch.setattr(sp.settings, "avito_proxy_url", None) + assert sp._avito_proxies() is None + + url = "http://u:p@mproxy.site:16928" + monkeypatch.setattr(sp.settings, "avito_proxy_url", url) + assert sp._avito_proxies() == {"http": url, "https": url}