diff --git a/tradein-mvp/backend/app/api/v1/admin.py b/tradein-mvp/backend/app/api/v1/admin.py index b26e3bf1..450dd0a7 100644 --- a/tradein-mvp/backend/app/api/v1/admin.py +++ b/tradein-mvp/backend/app/api/v1/admin.py @@ -28,9 +28,27 @@ from pydantic import BaseModel, Field, field_validator # (endpoint не прокинут через config, в отличие от SERP-классов и BaseScraper.__aenter__ # соседних провайдеров) — вне scope этой задачи (см. "только consume, не трогать # scraper_kit provider-логику"), фиксируем как follow-up. +# +# scraper_kit-миграция (#2397 slice A, эпик #2277): 5 debug city-sweep/full-load +# роутов (avito-city-sweep, cian-city-sweep, cian-full-load, yandex-city-sweep, +# yandex-full-load) переключены с app.services.scrape_pipeline (legacy) на +# scraper_kit.orchestration.pipeline — тот же DI-паттерн, что и app.scheduler_main +# ._run_kit_scheduler / scraper_kit.orchestration.scheduler._job_*: config/matcher/ +# enrichment/proxy_provider инжектируются явно вместо module-level импортов app.* +# внутри scrape_pipeline.py. shutdown_requested НЕ прокидывается (нет SIGTERM-drain +# семантики у admin BackgroundTasks — эквивалент дефолту lambda: False, поведение не +# меняется). Боевой scheduler.py (production sweep-cron) остаётся на legacy — это +# отдельный, ещё больший шаг (см. vault Event_Legacy_Scrapers_Partial_Deletion_Jul04). from scraper_kit.base import save_listings from scraper_kit.browser_fetcher import BrowserFetcher -from scraper_kit.orchestration.pipeline import DEFAULT_REGION_CODE +from scraper_kit.orchestration.pipeline import ( + DEFAULT_REGION_CODE, + run_avito_city_sweep, + run_cian_city_sweep, + run_cian_full_load, + run_yandex_city_sweep, + run_yandex_full_load, +) from scraper_kit.providers.avito.detail import fetch_detail, save_detail_enrichment from scraper_kit.providers.avito.houses import fetch_house_catalog, save_house_catalog_enrichment from scraper_kit.providers.avito.imv import ( @@ -54,14 +72,12 @@ from app.services import cian_session as cian_session_svc from app.services import scrape_runs as runs_mod from app.services.geocoder import geocode from app.services.scheduler import has_running_run -from app.services.scrape_pipeline import ( - run_avito_city_sweep, - run_cian_city_sweep, - run_cian_full_load, - run_yandex_city_sweep, - run_yandex_full_load, +from app.services.scraper_adapters import ( + RealEnrichmentJobs, + RealMatcherAdapter, + RealProxyProvider, + RealScraperConfig, ) -from app.services.scraper_adapters import RealMatcherAdapter, RealProxyProvider, RealScraperConfig from app.services.scraper_settings import get_scraper_delay, invalidate_cache from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper from app.tasks.avito_detail_backfill import run_avito_detail_backfill @@ -840,6 +856,10 @@ async def start_avito_city_sweep( await run_avito_city_sweep( sweep_db, run_id=run_id, + config=RealScraperConfig(), + matcher=RealMatcherAdapter(), + enrichment=RealEnrichmentJobs(), + proxy_provider=_kit_proxy_provider(), radius_m=payload.radius_m, pages_per_anchor=payload.pages_per_anchor, enrich_houses=payload.enrich_houses, @@ -930,6 +950,9 @@ async def start_cian_city_sweep( await run_cian_city_sweep( sweep_db, run_id=run_id, + config=RealScraperConfig(), + matcher=RealMatcherAdapter(), + proxy_provider=_kit_proxy_provider(), radius_m=payload.radius_m, pages_per_anchor=payload.pages_per_anchor, request_delay_sec=payload.request_delay_sec, @@ -1059,6 +1082,9 @@ async def start_cian_full_load( await run_cian_full_load( task_db, run_id=run_id, + config=RealScraperConfig(), + matcher=RealMatcherAdapter(), + proxy_provider=_kit_proxy_provider(), price_cap_per_bucket=payload.price_cap_per_bucket, request_delay_sec=payload.request_delay_sec, concurrency=payload.concurrency, @@ -1160,6 +1186,10 @@ async def start_yandex_full_load( await run_yandex_full_load( task_db, run_id=run_id, + config=RealScraperConfig(), + matcher=RealMatcherAdapter(), + enrichment=RealEnrichmentJobs(), + proxy_provider=_kit_proxy_provider(), price_cap_per_bucket=payload.price_cap_per_bucket, request_delay_sec=payload.request_delay_sec, concurrency=payload.concurrency, @@ -1224,6 +1254,10 @@ async def start_yandex_city_sweep( await run_yandex_city_sweep( sweep_db, run_id=run_id, + config=RealScraperConfig(), + matcher=RealMatcherAdapter(), + enrichment=RealEnrichmentJobs(), + proxy_provider=_kit_proxy_provider(), radius_m=payload.radius_m, pages_per_anchor=payload.pages_per_anchor, request_delay_sec=payload.request_delay_sec, diff --git a/tradein-mvp/backend/tests/test_cian_city_sweep.py b/tradein-mvp/backend/tests/test_cian_city_sweep.py index 2c839d4c..9b84de47 100644 --- a/tradein-mvp/backend/tests/test_cian_city_sweep.py +++ b/tradein-mvp/backend/tests/test_cian_city_sweep.py @@ -717,12 +717,14 @@ def test_cian_start_endpoint_validates_delay_too_low(app_with_admin) -> None: def test_cian_start_endpoint_ok(app_with_admin) -> None: - """Valid request → 200, run_id в ответе.""" + """Valid request → 200, run_id; kit sweep вызван с DI (config/matcher, БЕЗ enrichment).""" from unittest.mock import AsyncMock, patch + from app.services.scraper_adapters import RealMatcherAdapter, RealScraperConfig + with ( patch("app.services.scrape_runs.create_run", return_value=42), - patch("app.services.scrape_pipeline.run_cian_city_sweep", new_callable=AsyncMock), + patch("app.api.v1.admin.run_cian_city_sweep", new_callable=AsyncMock) as mock_sweep, ): r = app_with_admin.post( "/api/v1/admin/scrape/cian-city-sweep", @@ -735,6 +737,13 @@ def test_cian_start_endpoint_ok(app_with_admin) -> None: assert body["pages_per_anchor"] == 2 assert body["detail_top_n"] == 5 + mock_sweep.assert_called_once() + kw = mock_sweep.call_args.kwargs + assert isinstance(kw["config"], RealScraperConfig) + assert isinstance(kw["matcher"], RealMatcherAdapter) + assert "proxy_provider" in kw + assert "enrichment" not in kw + def test_cian_cancel_endpoint(app_with_admin) -> None: """Cancel endpoint возвращает run_id + cancelled=True.""" diff --git a/tradein-mvp/backend/tests/test_city_sweep.py b/tradein-mvp/backend/tests/test_city_sweep.py index 2dbfae1e..31684af7 100644 --- a/tradein-mvp/backend/tests/test_city_sweep.py +++ b/tradein-mvp/backend/tests/test_city_sweep.py @@ -313,11 +313,17 @@ def test_start_endpoint_validates_request_delay_too_low(app_with_admin) -> None: def test_start_endpoint_ok(app_with_admin) -> None: - """Valid request → 200, run_id в ответе.""" + """Valid request → 200, run_id в ответе; kit sweep вызван с DI (config/matcher/enrichment).""" + from app.services.scraper_adapters import ( + RealEnrichmentJobs, + RealMatcherAdapter, + RealScraperConfig, + ) + with ( patch("app.api.v1.admin.has_running_run", return_value=False), patch("app.services.scrape_runs.create_run", return_value=7), - patch("app.services.scrape_pipeline.run_avito_city_sweep", new_callable=AsyncMock), + patch("app.api.v1.admin.run_avito_city_sweep", new_callable=AsyncMock) as mock_sweep, ): r = app_with_admin.post( "/api/v1/admin/scrape/avito-city-sweep", @@ -329,6 +335,13 @@ def test_start_endpoint_ok(app_with_admin) -> None: assert body["status"] == "running" assert body["pages_per_anchor"] == 2 + mock_sweep.assert_called_once() + kw = mock_sweep.call_args.kwargs + assert isinstance(kw["config"], RealScraperConfig) + assert isinstance(kw["matcher"], RealMatcherAdapter) + assert isinstance(kw["enrichment"], RealEnrichmentJobs) + assert "proxy_provider" in kw + def test_cancel_endpoint_exists(app_with_admin) -> None: """Cancel endpoint returns run_id + cancelled flag.""" diff --git a/tradein-mvp/backend/tests/test_sweep_imv_phase.py b/tradein-mvp/backend/tests/test_sweep_imv_phase.py index 7b501727..7a577320 100644 --- a/tradein-mvp/backend/tests/test_sweep_imv_phase.py +++ b/tradein-mvp/backend/tests/test_sweep_imv_phase.py @@ -453,14 +453,24 @@ def test_city_sweep_start_request_enrich_imv_false() -> None: def test_sweep_endpoint_passes_enrich_imv(app_with_admin) -> None: - """POST /scrape/avito-city-sweep с enrich_imv=False → sweep вызывается без IMV.""" + """POST /scrape/avito-city-sweep с enrich_imv=False → sweep вызывается без IMV. + + kit sweep вызван с DI (config/matcher/enrichment) — mock должен перехватывать + имя, забинженное в app.api.v1.admin (kit-импорт), не устаревший scrape_pipeline. + """ + from app.services.scraper_adapters import ( + RealEnrichmentJobs, + RealMatcherAdapter, + RealScraperConfig, + ) + with ( patch("app.api.v1.admin.has_running_run", return_value=False), patch("app.services.scrape_runs.create_run", return_value=42), patch( - "app.services.scrape_pipeline.run_avito_city_sweep", + "app.api.v1.admin.run_avito_city_sweep", new_callable=AsyncMock, - ), + ) as mock_sweep, ): r = app_with_admin.post( "/api/v1/admin/scrape/avito-city-sweep", @@ -470,6 +480,14 @@ def test_sweep_endpoint_passes_enrich_imv(app_with_admin) -> None: body = r.json() assert body["run_id"] == 42 + mock_sweep.assert_called_once() + kw = mock_sweep.call_args.kwargs + assert isinstance(kw["config"], RealScraperConfig) + assert isinstance(kw["matcher"], RealMatcherAdapter) + assert isinstance(kw["enrichment"], RealEnrichmentJobs) + assert "proxy_provider" in kw + assert kw["enrich_imv"] is False + @pytest.fixture def app_with_admin(): diff --git a/tradein-mvp/backend/tests/test_yandex_city_sweep.py b/tradein-mvp/backend/tests/test_yandex_city_sweep.py index 3eeaa61a..32b31898 100644 --- a/tradein-mvp/backend/tests/test_yandex_city_sweep.py +++ b/tradein-mvp/backend/tests/test_yandex_city_sweep.py @@ -623,12 +623,18 @@ def test_yandex_start_endpoint_validates_delay_too_low(app_with_admin) -> None: def test_yandex_start_endpoint_ok(app_with_admin) -> None: - """Valid request → 200, run_id в ответе.""" + """Valid request → 200, run_id в ответе; kit sweep вызван с DI (config/matcher/enrichment).""" from unittest.mock import patch + from app.services.scraper_adapters import ( + RealEnrichmentJobs, + RealMatcherAdapter, + RealScraperConfig, + ) + with ( patch("app.services.scrape_runs.create_run", return_value=55), - patch("app.services.scrape_pipeline.run_yandex_city_sweep", new_callable=AsyncMock), + patch("app.api.v1.admin.run_yandex_city_sweep", new_callable=AsyncMock) as mock_sweep, ): r = app_with_admin.post( "/api/v1/admin/scrape/yandex-city-sweep", @@ -641,6 +647,13 @@ def test_yandex_start_endpoint_ok(app_with_admin) -> None: assert body["pages_per_anchor"] == 2 assert body["detail_top_n"] == 0 + mock_sweep.assert_called_once() + kw = mock_sweep.call_args.kwargs + assert isinstance(kw["config"], RealScraperConfig) + assert isinstance(kw["matcher"], RealMatcherAdapter) + assert isinstance(kw["enrichment"], RealEnrichmentJobs) + assert "proxy_provider" in kw + def test_yandex_cancel_endpoint(app_with_admin) -> None: """Cancel endpoint возвращает run_id + cancelled=True."""