Compare commits
No commits in common. "27ba08b821a1d02ade801d2fca29e6005cde8e03" and "155d5ec3a96a731295a26826a91ae35dbc7aaed1" have entirely different histories.
27ba08b821
...
155d5ec3a9
1 changed files with 3 additions and 10 deletions
|
|
@ -29,14 +29,6 @@ from scraper_kit.avito_exceptions import (
|
||||||
AvitoRateLimitedError,
|
AvitoRateLimitedError,
|
||||||
)
|
)
|
||||||
from scraper_kit.browser_fetcher import BrowserFetcher
|
from scraper_kit.browser_fetcher import BrowserFetcher
|
||||||
|
|
||||||
# #2397 slice B (эпик #2277 decommission scrape_pipeline.py, Part E): раньше
|
|
||||||
# _CHROME_HEADERS/_avito_proxies() импортировались из app.services.scrape_pipeline.
|
|
||||||
# kit уже вынес byte-идентичный dict (DOCUMENT_HEADERS) и ту же формулу
|
|
||||||
# {"http": url, "https": url} if url else None (http_proxies) в
|
|
||||||
# providers/_base.py (#2358 Foundation) — реюзаем вместо копии, разрывая
|
|
||||||
# зависимость от scrape_pipeline.py перед его удалением.
|
|
||||||
from scraper_kit.providers._base import DOCUMENT_HEADERS, http_proxies
|
|
||||||
from scraper_kit.providers.avito.detail import (
|
from scraper_kit.providers.avito.detail import (
|
||||||
fetch_detail,
|
fetch_detail,
|
||||||
research_in_session,
|
research_in_session,
|
||||||
|
|
@ -49,6 +41,7 @@ from sqlalchemy.orm import Session
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
from app.core.shutdown import shutdown_requested
|
from app.core.shutdown import shutdown_requested
|
||||||
from app.services import scrape_runs as runs_mod
|
from app.services import scrape_runs as runs_mod
|
||||||
|
from app.services.scrape_pipeline import _CHROME_HEADERS, _avito_proxies
|
||||||
from app.services.scraper_adapters import RealScraperConfig
|
from app.services.scraper_adapters import RealScraperConfig
|
||||||
|
|
||||||
# _AVITO_WARM_SEARCH_URL/build_warmed_session остаются legacy (#2310): kit's
|
# _AVITO_WARM_SEARCH_URL/build_warmed_session остаются legacy (#2310): kit's
|
||||||
|
|
@ -181,8 +174,8 @@ async def run_avito_detail_backfill(
|
||||||
session = AsyncSession(
|
session = AsyncSession(
|
||||||
impersonate="chrome120",
|
impersonate="chrome120",
|
||||||
timeout=25,
|
timeout=25,
|
||||||
headers=DOCUMENT_HEADERS,
|
headers=_CHROME_HEADERS,
|
||||||
proxies=http_proxies(settings.scraper_proxy_url),
|
proxies=_avito_proxies(),
|
||||||
)
|
)
|
||||||
scraper._cffi = session
|
scraper._cffi = session
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue