Merge pull request 'refactor(tradein/avito): avito_detail_backfill off scrape_pipeline helpers (#2397 slice B)' (#2399) from refactor/tradein-extract-avito-scrape-helpers into main
All checks were successful
Deploy Trade-In / changes (push) Successful in 10s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 1m32s
Deploy Trade-In / build-backend (push) Successful in 57s
Deploy Trade-In / deploy (push) Successful in 1m30s
All checks were successful
Deploy Trade-In / changes (push) Successful in 10s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / test (push) Successful in 1m32s
Deploy Trade-In / build-backend (push) Successful in 57s
Deploy Trade-In / deploy (push) Successful in 1m30s
This commit is contained in:
commit
27ba08b821
1 changed files with 10 additions and 3 deletions
|
|
@ -29,6 +29,14 @@ from scraper_kit.avito_exceptions import (
|
|||
AvitoRateLimitedError,
|
||||
)
|
||||
from scraper_kit.browser_fetcher import BrowserFetcher
|
||||
|
||||
# #2397 slice B (эпик #2277 decommission scrape_pipeline.py, Part E): раньше
|
||||
# _CHROME_HEADERS/_avito_proxies() импортировались из app.services.scrape_pipeline.
|
||||
# kit уже вынес byte-идентичный dict (DOCUMENT_HEADERS) и ту же формулу
|
||||
# {"http": url, "https": url} if url else None (http_proxies) в
|
||||
# providers/_base.py (#2358 Foundation) — реюзаем вместо копии, разрывая
|
||||
# зависимость от scrape_pipeline.py перед его удалением.
|
||||
from scraper_kit.providers._base import DOCUMENT_HEADERS, http_proxies
|
||||
from scraper_kit.providers.avito.detail import (
|
||||
fetch_detail,
|
||||
research_in_session,
|
||||
|
|
@ -41,7 +49,6 @@ from sqlalchemy.orm import Session
|
|||
from app.core.config import settings
|
||||
from app.core.shutdown import shutdown_requested
|
||||
from app.services import scrape_runs as runs_mod
|
||||
from app.services.scrape_pipeline import _CHROME_HEADERS, _avito_proxies
|
||||
from app.services.scraper_adapters import RealScraperConfig
|
||||
|
||||
# _AVITO_WARM_SEARCH_URL/build_warmed_session остаются legacy (#2310): kit's
|
||||
|
|
@ -174,8 +181,8 @@ async def run_avito_detail_backfill(
|
|||
session = AsyncSession(
|
||||
impersonate="chrome120",
|
||||
timeout=25,
|
||||
headers=_CHROME_HEADERS,
|
||||
proxies=_avito_proxies(),
|
||||
headers=DOCUMENT_HEADERS,
|
||||
proxies=http_proxies(settings.scraper_proxy_url),
|
||||
)
|
||||
scraper._cffi = session
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue