refactor(tradein/admin): debug sweep endpoints на kit orchestration (#2397 slice A)
Мигрирует 5 admin debug-роутов с legacy app.services.scrape_pipeline на scraper_kit.orchestration.pipeline (эпик #2277 decommission, umbrella #2397): - POST /scrape/avito-city-sweep -> run_avito_city_sweep - POST /scrape/cian-city-sweep -> run_cian_city_sweep - POST /scrape/cian-full-load -> run_cian_full_load - POST /scrape/yandex-city-sweep -> run_yandex_city_sweep - POST /scrape/yandex-full-load -> run_yandex_full_load DI-паттерн зеркалит app.scheduler_main._run_kit_scheduler / scraper_kit.orchestration.scheduler._job_* (уже используется в этом же файле для Group A #2305 debug-роутов): config=RealScraperConfig(), matcher=RealMatcherAdapter(), proxy_provider=_kit_proxy_provider() для всех пяти + enrichment=RealEnrichmentJobs() там, где kit-сигнатура его требует (avito_city_sweep, yandex_city_sweep, yandex_full_load — cian-варианты enrichment не принимают). shutdown_requested не прокидывается — у admin BackgroundTasks нет SIGTERM-drain семантики, дефолт kit (lambda: False) эквивалентен прежнему поведению. Разблокирует финальное удаление зависимости admin.py от scrape_pipeline.py. Production-scheduler (app/services/scheduler.py, USE_KIT_SCHEDULER=False ship-dark) остаётся на legacy — это отдельный, более крупный шаг миграции SERP-оркестрации.
This commit is contained in:
parent
27ba08b821
commit
42b1ca6d7f
1 changed files with 42 additions and 8 deletions
|
|
@ -28,9 +28,27 @@ from pydantic import BaseModel, Field, field_validator
|
||||||
# (endpoint не прокинут через config, в отличие от SERP-классов и BaseScraper.__aenter__
|
# (endpoint не прокинут через config, в отличие от SERP-классов и BaseScraper.__aenter__
|
||||||
# соседних провайдеров) — вне scope этой задачи (см. "только consume, не трогать
|
# соседних провайдеров) — вне scope этой задачи (см. "только consume, не трогать
|
||||||
# scraper_kit provider-логику"), фиксируем как follow-up.
|
# scraper_kit provider-логику"), фиксируем как follow-up.
|
||||||
|
#
|
||||||
|
# scraper_kit-миграция (#2397 slice A, эпик #2277): 5 debug city-sweep/full-load
|
||||||
|
# роутов (avito-city-sweep, cian-city-sweep, cian-full-load, yandex-city-sweep,
|
||||||
|
# yandex-full-load) переключены с app.services.scrape_pipeline (legacy) на
|
||||||
|
# scraper_kit.orchestration.pipeline — тот же DI-паттерн, что и app.scheduler_main
|
||||||
|
# ._run_kit_scheduler / scraper_kit.orchestration.scheduler._job_*: config/matcher/
|
||||||
|
# enrichment/proxy_provider инжектируются явно вместо module-level импортов app.*
|
||||||
|
# внутри scrape_pipeline.py. shutdown_requested НЕ прокидывается (нет SIGTERM-drain
|
||||||
|
# семантики у admin BackgroundTasks — эквивалент дефолту lambda: False, поведение не
|
||||||
|
# меняется). Боевой scheduler.py (production sweep-cron) остаётся на legacy — это
|
||||||
|
# отдельный, ещё больший шаг (см. vault Event_Legacy_Scrapers_Partial_Deletion_Jul04).
|
||||||
from scraper_kit.base import save_listings
|
from scraper_kit.base import save_listings
|
||||||
from scraper_kit.browser_fetcher import BrowserFetcher
|
from scraper_kit.browser_fetcher import BrowserFetcher
|
||||||
from scraper_kit.orchestration.pipeline import DEFAULT_REGION_CODE
|
from scraper_kit.orchestration.pipeline import (
|
||||||
|
DEFAULT_REGION_CODE,
|
||||||
|
run_avito_city_sweep,
|
||||||
|
run_cian_city_sweep,
|
||||||
|
run_cian_full_load,
|
||||||
|
run_yandex_city_sweep,
|
||||||
|
run_yandex_full_load,
|
||||||
|
)
|
||||||
from scraper_kit.providers.avito.detail import fetch_detail, save_detail_enrichment
|
from scraper_kit.providers.avito.detail import fetch_detail, save_detail_enrichment
|
||||||
from scraper_kit.providers.avito.houses import fetch_house_catalog, save_house_catalog_enrichment
|
from scraper_kit.providers.avito.houses import fetch_house_catalog, save_house_catalog_enrichment
|
||||||
from scraper_kit.providers.avito.imv import (
|
from scraper_kit.providers.avito.imv import (
|
||||||
|
|
@ -54,14 +72,12 @@ from app.services import cian_session as cian_session_svc
|
||||||
from app.services import scrape_runs as runs_mod
|
from app.services import scrape_runs as runs_mod
|
||||||
from app.services.geocoder import geocode
|
from app.services.geocoder import geocode
|
||||||
from app.services.scheduler import has_running_run
|
from app.services.scheduler import has_running_run
|
||||||
from app.services.scrape_pipeline import (
|
from app.services.scraper_adapters import (
|
||||||
run_avito_city_sweep,
|
RealEnrichmentJobs,
|
||||||
run_cian_city_sweep,
|
RealMatcherAdapter,
|
||||||
run_cian_full_load,
|
RealProxyProvider,
|
||||||
run_yandex_city_sweep,
|
RealScraperConfig,
|
||||||
run_yandex_full_load,
|
|
||||||
)
|
)
|
||||||
from app.services.scraper_adapters import RealMatcherAdapter, RealProxyProvider, RealScraperConfig
|
|
||||||
from app.services.scraper_settings import get_scraper_delay, invalidate_cache
|
from app.services.scraper_settings import get_scraper_delay, invalidate_cache
|
||||||
from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
|
from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
|
||||||
from app.tasks.avito_detail_backfill import run_avito_detail_backfill
|
from app.tasks.avito_detail_backfill import run_avito_detail_backfill
|
||||||
|
|
@ -840,6 +856,10 @@ async def start_avito_city_sweep(
|
||||||
await run_avito_city_sweep(
|
await run_avito_city_sweep(
|
||||||
sweep_db,
|
sweep_db,
|
||||||
run_id=run_id,
|
run_id=run_id,
|
||||||
|
config=RealScraperConfig(),
|
||||||
|
matcher=RealMatcherAdapter(),
|
||||||
|
enrichment=RealEnrichmentJobs(),
|
||||||
|
proxy_provider=_kit_proxy_provider(),
|
||||||
radius_m=payload.radius_m,
|
radius_m=payload.radius_m,
|
||||||
pages_per_anchor=payload.pages_per_anchor,
|
pages_per_anchor=payload.pages_per_anchor,
|
||||||
enrich_houses=payload.enrich_houses,
|
enrich_houses=payload.enrich_houses,
|
||||||
|
|
@ -930,6 +950,9 @@ async def start_cian_city_sweep(
|
||||||
await run_cian_city_sweep(
|
await run_cian_city_sweep(
|
||||||
sweep_db,
|
sweep_db,
|
||||||
run_id=run_id,
|
run_id=run_id,
|
||||||
|
config=RealScraperConfig(),
|
||||||
|
matcher=RealMatcherAdapter(),
|
||||||
|
proxy_provider=_kit_proxy_provider(),
|
||||||
radius_m=payload.radius_m,
|
radius_m=payload.radius_m,
|
||||||
pages_per_anchor=payload.pages_per_anchor,
|
pages_per_anchor=payload.pages_per_anchor,
|
||||||
request_delay_sec=payload.request_delay_sec,
|
request_delay_sec=payload.request_delay_sec,
|
||||||
|
|
@ -1059,6 +1082,9 @@ async def start_cian_full_load(
|
||||||
await run_cian_full_load(
|
await run_cian_full_load(
|
||||||
task_db,
|
task_db,
|
||||||
run_id=run_id,
|
run_id=run_id,
|
||||||
|
config=RealScraperConfig(),
|
||||||
|
matcher=RealMatcherAdapter(),
|
||||||
|
proxy_provider=_kit_proxy_provider(),
|
||||||
price_cap_per_bucket=payload.price_cap_per_bucket,
|
price_cap_per_bucket=payload.price_cap_per_bucket,
|
||||||
request_delay_sec=payload.request_delay_sec,
|
request_delay_sec=payload.request_delay_sec,
|
||||||
concurrency=payload.concurrency,
|
concurrency=payload.concurrency,
|
||||||
|
|
@ -1160,6 +1186,10 @@ async def start_yandex_full_load(
|
||||||
await run_yandex_full_load(
|
await run_yandex_full_load(
|
||||||
task_db,
|
task_db,
|
||||||
run_id=run_id,
|
run_id=run_id,
|
||||||
|
config=RealScraperConfig(),
|
||||||
|
matcher=RealMatcherAdapter(),
|
||||||
|
enrichment=RealEnrichmentJobs(),
|
||||||
|
proxy_provider=_kit_proxy_provider(),
|
||||||
price_cap_per_bucket=payload.price_cap_per_bucket,
|
price_cap_per_bucket=payload.price_cap_per_bucket,
|
||||||
request_delay_sec=payload.request_delay_sec,
|
request_delay_sec=payload.request_delay_sec,
|
||||||
concurrency=payload.concurrency,
|
concurrency=payload.concurrency,
|
||||||
|
|
@ -1224,6 +1254,10 @@ async def start_yandex_city_sweep(
|
||||||
await run_yandex_city_sweep(
|
await run_yandex_city_sweep(
|
||||||
sweep_db,
|
sweep_db,
|
||||||
run_id=run_id,
|
run_id=run_id,
|
||||||
|
config=RealScraperConfig(),
|
||||||
|
matcher=RealMatcherAdapter(),
|
||||||
|
enrichment=RealEnrichmentJobs(),
|
||||||
|
proxy_provider=_kit_proxy_provider(),
|
||||||
radius_m=payload.radius_m,
|
radius_m=payload.radius_m,
|
||||||
pages_per_anchor=payload.pages_per_anchor,
|
pages_per_anchor=payload.pages_per_anchor,
|
||||||
request_delay_sec=payload.request_delay_sec,
|
request_delay_sec=payload.request_delay_sec,
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue