From 8ca529ece085b8d45734b95fe38d9ee3f38be37a Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 11:55:24 +0300 Subject: [PATCH] refactor(tradein/avito): avito_detail_backfill off scrape_pipeline helpers (#2397 slice B) app/tasks/avito_detail_backfill.py imported _CHROME_HEADERS/_avito_proxies from the legacy app/services/scrape_pipeline.py (the only two symbols it needed from that module). scrape_pipeline.py is slated for wholesale deletion in Part E of epic #2277 -- this severs the last dependency so that deletion won't break the backfill task. Source chosen: kit-reuse, not relocate. scraper_kit/providers/_base.py (#2358 Foundation) already extracted the identical building blocks while deduplicating providers/avito/{serp,detail,imv}.py: - DOCUMENT_HEADERS -- byte-identical dict to _CHROME_HEADERS (same 8 keys/values, Accept/Accept-Language/Cache-Control/Sec-Fetch-*/Upgrade-Insecure-Requests). - http_proxies(proxy_url) -- same formula as _avito_proxies(): {"http": url, "https": url} if url else None, just parameterized instead of reading settings.scraper_proxy_url internally (both resolve the identical settings singleton in production, so behavior is unchanged). Kept the AsyncSession(...) construction inline (didn't switch to kit's build_document_session helper) to preserve the existing `app.tasks.avito_detail_backfill.AsyncSession` mock seam used by tests/tasks/test_avito_detail_backfill.py -- swapping to the helper would call curl_cffi's AsyncSession via scraper_kit.providers._base instead, silently bypassing that patch target. scrape_pipeline.py itself is untouched (kept its own copies, per Part E plan). Full backend suite: 3276 passed, 1 known-unrelated fail (test_search_cache_hit, #2208), 6 skipped. --- .../backend/app/tasks/avito_detail_backfill.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py index 72133d59..a15dea3f 100644 --- a/tradein-mvp/backend/app/tasks/avito_detail_backfill.py +++ b/tradein-mvp/backend/app/tasks/avito_detail_backfill.py @@ -29,6 +29,14 @@ from scraper_kit.avito_exceptions import ( AvitoRateLimitedError, ) from scraper_kit.browser_fetcher import BrowserFetcher + +# #2397 slice B (эпик #2277 decommission scrape_pipeline.py, Part E): раньше +# _CHROME_HEADERS/_avito_proxies() импортировались из app.services.scrape_pipeline. +# kit уже вынес byte-идентичный dict (DOCUMENT_HEADERS) и ту же формулу +# {"http": url, "https": url} if url else None (http_proxies) в +# providers/_base.py (#2358 Foundation) — реюзаем вместо копии, разрывая +# зависимость от scrape_pipeline.py перед его удалением. +from scraper_kit.providers._base import DOCUMENT_HEADERS, http_proxies from scraper_kit.providers.avito.detail import ( fetch_detail, research_in_session, @@ -41,7 +49,6 @@ from sqlalchemy.orm import Session from app.core.config import settings from app.core.shutdown import shutdown_requested from app.services import scrape_runs as runs_mod -from app.services.scrape_pipeline import _CHROME_HEADERS, _avito_proxies from app.services.scraper_adapters import RealScraperConfig # _AVITO_WARM_SEARCH_URL/build_warmed_session остаются legacy (#2310): kit's @@ -174,8 +181,8 @@ async def run_avito_detail_backfill( session = AsyncSession( impersonate="chrome120", timeout=25, - headers=_CHROME_HEADERS, - proxies=_avito_proxies(), + headers=DOCUMENT_HEADERS, + proxies=http_proxies(settings.scraper_proxy_url), ) scraper._cffi = session -- 2.45.3