Merge pull request 'refactor(tradein/admin): debug sweep endpoints на kit orchestration (#2397 slice A)' (#2400) from refactor/tradein-admin-debug-sweeps-to-kit into main

tradein-mvp/backend/app/api/v1/admin.py

@@ -28,9 +28,27 @@ from pydantic import BaseModel, Field, field_validator
 # (endpoint не прокинут через config, в отличие от SERP-классов и BaseScraper.__aenter__
 # соседних провайдеров) — вне scope этой задачи (см. "только consume, не трогать
 # scraper_kit provider-логику"), фиксируем как follow-up.
+#
+# scraper_kit-миграция (#2397 slice A, эпик #2277): 5 debug city-sweep/full-load
+# роутов (avito-city-sweep, cian-city-sweep, cian-full-load, yandex-city-sweep,
+# yandex-full-load) переключены с app.services.scrape_pipeline (legacy) на
+# scraper_kit.orchestration.pipeline — тот же DI-паттерн, что и app.scheduler_main
+# ._run_kit_scheduler / scraper_kit.orchestration.scheduler._job_*: config/matcher/
+# enrichment/proxy_provider инжектируются явно вместо module-level импортов app.*
+# внутри scrape_pipeline.py. shutdown_requested НЕ прокидывается (нет SIGTERM-drain
+# семантики у admin BackgroundTasks — эквивалент дефолту lambda: False, поведение не
+# меняется). Боевой scheduler.py (production sweep-cron) остаётся на legacy — это
+# отдельный, ещё больший шаг (см. vault Event_Legacy_Scrapers_Partial_Deletion_Jul04).
 from scraper_kit.base import save_listings
 from scraper_kit.browser_fetcher import BrowserFetcher
-from scraper_kit.orchestration.pipeline import DEFAULT_REGION_CODE
+from scraper_kit.orchestration.pipeline import (
+    DEFAULT_REGION_CODE,
+    run_avito_city_sweep,
+    run_cian_city_sweep,
+    run_cian_full_load,
+    run_yandex_city_sweep,
+    run_yandex_full_load,
+)
 from scraper_kit.providers.avito.detail import fetch_detail, save_detail_enrichment
 from scraper_kit.providers.avito.houses import fetch_house_catalog, save_house_catalog_enrichment
 from scraper_kit.providers.avito.imv import (
@@ -54,14 +72,12 @@ from app.services import cian_session as cian_session_svc
 from app.services import scrape_runs as runs_mod
 from app.services.geocoder import geocode
 from app.services.scheduler import has_running_run
-from app.services.scrape_pipeline import (
+from app.services.scraper_adapters import (
-    run_avito_city_sweep,
+    RealEnrichmentJobs,
-    run_cian_city_sweep,
+    RealMatcherAdapter,
-    run_cian_full_load,
+    RealProxyProvider,
-    run_yandex_city_sweep,
+    RealScraperConfig,
-    run_yandex_full_load,
 )
-from app.services.scraper_adapters import RealMatcherAdapter, RealProxyProvider, RealScraperConfig
 from app.services.scraper_settings import get_scraper_delay, invalidate_cache
 from app.services.scrapers.yandex_newbuilding import YandexNewbuildingScraper
 from app.tasks.avito_detail_backfill import run_avito_detail_backfill
@@ -840,6 +856,10 @@ async def start_avito_city_sweep(
             await run_avito_city_sweep(
                 sweep_db,
                 run_id=run_id,
+                config=RealScraperConfig(),
+                matcher=RealMatcherAdapter(),
+                enrichment=RealEnrichmentJobs(),
+                proxy_provider=_kit_proxy_provider(),
                 radius_m=payload.radius_m,
                 pages_per_anchor=payload.pages_per_anchor,
                 enrich_houses=payload.enrich_houses,
@@ -930,6 +950,9 @@ async def start_cian_city_sweep(
             await run_cian_city_sweep(
                 sweep_db,
                 run_id=run_id,
+                config=RealScraperConfig(),
+                matcher=RealMatcherAdapter(),
+                proxy_provider=_kit_proxy_provider(),
                 radius_m=payload.radius_m,
                 pages_per_anchor=payload.pages_per_anchor,
                 request_delay_sec=payload.request_delay_sec,
@@ -1059,6 +1082,9 @@ async def start_cian_full_load(
             await run_cian_full_load(
                 task_db,
                 run_id=run_id,
+                config=RealScraperConfig(),
+                matcher=RealMatcherAdapter(),
+                proxy_provider=_kit_proxy_provider(),
                 price_cap_per_bucket=payload.price_cap_per_bucket,
                 request_delay_sec=payload.request_delay_sec,
                 concurrency=payload.concurrency,
@@ -1160,6 +1186,10 @@ async def start_yandex_full_load(
             await run_yandex_full_load(
                 task_db,
                 run_id=run_id,
+                config=RealScraperConfig(),
+                matcher=RealMatcherAdapter(),
+                enrichment=RealEnrichmentJobs(),
+                proxy_provider=_kit_proxy_provider(),
                 price_cap_per_bucket=payload.price_cap_per_bucket,
                 request_delay_sec=payload.request_delay_sec,
                 concurrency=payload.concurrency,
@@ -1224,6 +1254,10 @@ async def start_yandex_city_sweep(
             await run_yandex_city_sweep(
                 sweep_db,
                 run_id=run_id,
+                config=RealScraperConfig(),
+                matcher=RealMatcherAdapter(),
+                enrichment=RealEnrichmentJobs(),
+                proxy_provider=_kit_proxy_provider(),
                 radius_m=payload.radius_m,
                 pages_per_anchor=payload.pages_per_anchor,
                 request_delay_sec=payload.request_delay_sec,

tradein-mvp/backend/tests/test_cian_city_sweep.py

@@ -717,12 +717,14 @@ def test_cian_start_endpoint_validates_delay_too_low(app_with_admin) -> None:
 
 
 def test_cian_start_endpoint_ok(app_with_admin) -> None:
-    """Valid request → 200, run_id в ответе."""
+    """Valid request → 200, run_id; kit sweep вызван с DI (config/matcher, БЕЗ enrichment)."""
     from unittest.mock import AsyncMock, patch
 
+    from app.services.scraper_adapters import RealMatcherAdapter, RealScraperConfig
+
     with (
         patch("app.services.scrape_runs.create_run", return_value=42),
-        patch("app.services.scrape_pipeline.run_cian_city_sweep", new_callable=AsyncMock),
+        patch("app.api.v1.admin.run_cian_city_sweep", new_callable=AsyncMock) as mock_sweep,
     ):
         r = app_with_admin.post(
             "/api/v1/admin/scrape/cian-city-sweep",
@@ -735,6 +737,13 @@ def test_cian_start_endpoint_ok(app_with_admin) -> None:
     assert body["pages_per_anchor"] == 2
     assert body["detail_top_n"] == 5
 
+    mock_sweep.assert_called_once()
+    kw = mock_sweep.call_args.kwargs
+    assert isinstance(kw["config"], RealScraperConfig)
+    assert isinstance(kw["matcher"], RealMatcherAdapter)
+    assert "proxy_provider" in kw
+    assert "enrichment" not in kw
+
 
 def test_cian_cancel_endpoint(app_with_admin) -> None:
     """Cancel endpoint возвращает run_id + cancelled=True."""

tradein-mvp/backend/tests/test_city_sweep.py

@@ -313,11 +313,17 @@ def test_start_endpoint_validates_request_delay_too_low(app_with_admin) -> None:
 
 
 def test_start_endpoint_ok(app_with_admin) -> None:
-    """Valid request → 200, run_id в ответе."""
+    """Valid request → 200, run_id в ответе; kit sweep вызван с DI (config/matcher/enrichment)."""
+    from app.services.scraper_adapters import (
+        RealEnrichmentJobs,
+        RealMatcherAdapter,
+        RealScraperConfig,
+    )
+
     with (
         patch("app.api.v1.admin.has_running_run", return_value=False),
         patch("app.services.scrape_runs.create_run", return_value=7),
-        patch("app.services.scrape_pipeline.run_avito_city_sweep", new_callable=AsyncMock),
+        patch("app.api.v1.admin.run_avito_city_sweep", new_callable=AsyncMock) as mock_sweep,
     ):
         r = app_with_admin.post(
             "/api/v1/admin/scrape/avito-city-sweep",
@@ -329,6 +335,13 @@ def test_start_endpoint_ok(app_with_admin) -> None:
     assert body["status"] == "running"
     assert body["pages_per_anchor"] == 2
 
+    mock_sweep.assert_called_once()
+    kw = mock_sweep.call_args.kwargs
+    assert isinstance(kw["config"], RealScraperConfig)
+    assert isinstance(kw["matcher"], RealMatcherAdapter)
+    assert isinstance(kw["enrichment"], RealEnrichmentJobs)
+    assert "proxy_provider" in kw
+
 
 def test_cancel_endpoint_exists(app_with_admin) -> None:
     """Cancel endpoint returns run_id + cancelled flag."""

tradein-mvp/backend/tests/test_sweep_imv_phase.py

@@ -453,14 +453,24 @@ def test_city_sweep_start_request_enrich_imv_false() -> None:
 
 
 def test_sweep_endpoint_passes_enrich_imv(app_with_admin) -> None:
-    """POST /scrape/avito-city-sweep с enrich_imv=False → sweep вызывается без IMV."""
+    """POST /scrape/avito-city-sweep с enrich_imv=False → sweep вызывается без IMV.
+
+    kit sweep вызван с DI (config/matcher/enrichment) — mock должен перехватывать
+    имя, забинженное в app.api.v1.admin (kit-импорт), не устаревший scrape_pipeline.
+    """
+    from app.services.scraper_adapters import (
+        RealEnrichmentJobs,
+        RealMatcherAdapter,
+        RealScraperConfig,
+    )
+
     with (
         patch("app.api.v1.admin.has_running_run", return_value=False),
         patch("app.services.scrape_runs.create_run", return_value=42),
         patch(
-            "app.services.scrape_pipeline.run_avito_city_sweep",
+            "app.api.v1.admin.run_avito_city_sweep",
             new_callable=AsyncMock,
-        ),
+        ) as mock_sweep,
     ):
         r = app_with_admin.post(
             "/api/v1/admin/scrape/avito-city-sweep",
@@ -470,6 +480,14 @@ def test_sweep_endpoint_passes_enrich_imv(app_with_admin) -> None:
     body = r.json()
     assert body["run_id"] == 42
 
+    mock_sweep.assert_called_once()
+    kw = mock_sweep.call_args.kwargs
+    assert isinstance(kw["config"], RealScraperConfig)
+    assert isinstance(kw["matcher"], RealMatcherAdapter)
+    assert isinstance(kw["enrichment"], RealEnrichmentJobs)
+    assert "proxy_provider" in kw
+    assert kw["enrich_imv"] is False
+
 
 @pytest.fixture
 def app_with_admin():

tradein-mvp/backend/tests/test_yandex_city_sweep.py

@@ -623,12 +623,18 @@ def test_yandex_start_endpoint_validates_delay_too_low(app_with_admin) -> None:
 
 
 def test_yandex_start_endpoint_ok(app_with_admin) -> None:
-    """Valid request → 200, run_id в ответе."""
+    """Valid request → 200, run_id в ответе; kit sweep вызван с DI (config/matcher/enrichment)."""
     from unittest.mock import patch
 
+    from app.services.scraper_adapters import (
+        RealEnrichmentJobs,
+        RealMatcherAdapter,
+        RealScraperConfig,
+    )
+
     with (
         patch("app.services.scrape_runs.create_run", return_value=55),
-        patch("app.services.scrape_pipeline.run_yandex_city_sweep", new_callable=AsyncMock),
+        patch("app.api.v1.admin.run_yandex_city_sweep", new_callable=AsyncMock) as mock_sweep,
     ):
         r = app_with_admin.post(
             "/api/v1/admin/scrape/yandex-city-sweep",
@@ -641,6 +647,13 @@ def test_yandex_start_endpoint_ok(app_with_admin) -> None:
     assert body["pages_per_anchor"] == 2
     assert body["detail_top_n"] == 0
 
+    mock_sweep.assert_called_once()
+    kw = mock_sweep.call_args.kwargs
+    assert isinstance(kw["config"], RealScraperConfig)
+    assert isinstance(kw["matcher"], RealMatcherAdapter)
+    assert isinstance(kw["enrichment"], RealEnrichmentJobs)
+    assert "proxy_provider" in kw
+
 
 def test_yandex_cancel_endpoint(app_with_admin) -> None:
     """Cancel endpoint возвращает run_id + cancelled=True."""