Merge pull request 'refactor(tradein/tasks): yandex_newbuilding_sweep на kit + config= (#2397 Part D5)' (#2412) from refactor/tradein-partD5-yandex-nb-sweep-kit into main

tradein-mvp/backend/app/tasks/yandex_newbuilding_sweep.py

@@ -46,6 +46,7 @@ from dataclasses import dataclass, field, fields
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
+from app.services.scraper_adapters import RealScraperConfig
 from app.services.scraper_settings import get_scraper_delay
 
 logger = logging.getLogger(__name__)
@@ -221,16 +222,17 @@ async def enrich_yandex_newbuilding_sweep(
     Returns:
         YandexNewbuildingSweepResult со счётчиками.
     """
-    # NOT migrated to scraper_kit (issue #2310, Group C): both
-    # scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper.fetch_jk()
-    # AND resolve_yandex_jk_slug() construct BrowserFetcher(source="yandex")
-    # WITHOUT the now-mandatory endpoint= kwarg — every call raises TypeError,
-    # with no caller-side fix possible (the kit provider functions don't expose
-    # a config/endpoint hook at all). This is issue #2322, verified STILL OPEN
-    # by reading the actual provider source (not just the issue's open/closed
-    # status) at the time of this migration. Left on legacy entirely, exactly
-    # like Group A (#2305) did for the same bug in admin.py's debug endpoints.
-    from app.services.scrapers.yandex_newbuilding import (
+    # Migrated to scraper_kit (#2397 Part D5, issue #2310 Group C): #2322
+    # (BrowserFetcher(source="yandex") constructed без endpoint= kwarg) was
+    # fixed by PR #2356 — both YandexNewbuildingScraper.fetch_jk() и
+    # resolve_yandex_jk_slug() now accept config=ScraperConfig и пробрасывают
+    # config.browser_http_endpoint в BrowserFetcher. RealScraperConfig()
+    # (см. import выше) обязателен на ОБОИХ call site — без него
+    # BrowserFetcher.__aenter__ падает AssertionError (endpoint не задан).
+    # Lazy import (не module top) — как и раньше, чтобы тяжёлый browser/parse
+    # стек грузился только когда sweep реально запускается (unit-тесты
+    # патчат эти имена cheap-ly через mocker без импорта selectolax и т.п.).
+    from scraper_kit.providers.yandex.newbuilding import (
         YandexNewbuildingScraper,
         resolve_yandex_jk_slug,
     )
@@ -301,7 +303,9 @@ async def enrich_yandex_newbuilding_sweep(
                 continue
 
             try:
-                resolved = await resolve_yandex_jk_slug(ext_id, city=city)
+                resolved = await resolve_yandex_jk_slug(
+                    ext_id, city=city, config=RealScraperConfig()
+                )
             except Exception as exc:
                 logger.warning(
                     "resolve_yandex_jk_slug house_id=%s ext_id=%s raised: %s",
@@ -350,7 +354,7 @@ async def enrich_yandex_newbuilding_sweep(
         # ── Fetch через BrowserFetcher ────────────────────────────────────
         info = None
         try:
-            scraper = YandexNewbuildingScraper()
+            scraper = YandexNewbuildingScraper(config=RealScraperConfig())
             info = await scraper.fetch_jk(jk_slug=jk_slug, jk_id=ext_id, city=city)
         except Exception as exc:
             logger.warning(

tradein-mvp/backend/tests/tasks/test_yandex_newbuilding_sweep.py

@@ -192,7 +192,7 @@ async def test_sweep_with_existing_slug_upserts_enrichment():
         ]
     )
 
-    from app.services.scrapers.yandex_newbuilding import YandexNewbuildingInfo
+    from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingInfo
 
     fake_info = YandexNewbuildingInfo(
         ext_id="999",
@@ -207,7 +207,9 @@ async def test_sweep_with_existing_slug_upserts_enrichment():
         text_reviews_count=25,
     )
 
-    with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
+    from app.services.scraper_adapters import RealScraperConfig
+
+    with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper:
         scraper_instance = MagicMock()
         scraper_instance.fetch_jk = AsyncMock(return_value=fake_info)
         mock_scraper.return_value = scraper_instance
@@ -224,6 +226,12 @@ async def test_sweep_with_existing_slug_upserts_enrichment():
     # house_class не распарсен в этой фикстуре (None) — UPDATE houses.house_class не должен
     # выполниться вовсе (Bug 1 guard: лишний UPDATE с NULL пропускается).
     assert 10 not in db.house_class_updates
+    # #2397 Part D5 regression guard: kit YandexNewbuildingScraper.fetch_jk builds
+    # BrowserFetcher(endpoint=self._config.browser_http_endpoint) — without
+    # config=RealScraperConfig() threaded at the call site, BrowserFetcher.__aenter__
+    # raises AssertionError (missing endpoint). call_count alone wouldn't catch a
+    # future drop of that kwarg (mirrors #2306/#2310's isinstance-call_kwargs pattern).
+    assert isinstance(mock_scraper.call_args.kwargs.get("config"), RealScraperConfig)
 
 
 @pytest.mark.asyncio
@@ -244,7 +252,7 @@ async def test_sweep_persists_house_class_when_parsed():
         ]
     )
 
-    from app.services.scrapers.yandex_newbuilding import YandexNewbuildingInfo
+    from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingInfo
 
     fake_info = YandexNewbuildingInfo(
         ext_id="1000",
@@ -254,7 +262,7 @@ async def test_sweep_persists_house_class_when_parsed():
         house_class="comfort_plus",
     )
 
-    with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
+    with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper:
         scraper_instance = MagicMock()
         scraper_instance.fetch_jk = AsyncMock(return_value=fake_info)
         mock_scraper.return_value = scraper_instance
@@ -279,7 +287,9 @@ async def test_sweep_resolves_slug_when_missing():
         ]
     )
 
-    from app.services.scrapers.yandex_newbuilding import YandexNewbuildingInfo
+    from scraper_kit.providers.yandex.newbuilding import YandexNewbuildingInfo
+
+    from app.services.scraper_adapters import RealScraperConfig
 
     fake_info = YandexNewbuildingInfo(
         ext_id="888",
@@ -292,11 +302,11 @@ async def test_sweep_resolves_slug_when_missing():
 
     with (
         patch(
-            "app.services.scrapers.yandex_newbuilding.resolve_yandex_jk_slug",
+            "scraper_kit.providers.yandex.newbuilding.resolve_yandex_jk_slug",
             new_callable=AsyncMock,
             return_value="testjk",
         ) as mock_resolve,
-        patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper,
+        patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper,
     ):
         scraper_instance = MagicMock()
         scraper_instance.fetch_jk = AsyncMock(return_value=fake_info)
@@ -304,7 +314,14 @@ async def test_sweep_resolves_slug_when_missing():
 
         result = await enrich_yandex_newbuilding_sweep(db, limit=5, request_delay_sec=0)
 
-    mock_resolve.assert_called_once_with("888", city="ekaterinburg")
+    # resolve_yandex_jk_slug теперь принимает config= (#2322 fix) — сравниваем
+    # позиционный arg + city= отдельно от config, т.к. RealScraperConfig
+    # instance-equality не поддерживает (нет __eq__); проверяем isinstance,
+    # тот же regression-guard паттерн что и для YandexNewbuildingScraper выше.
+    resolve_args, resolve_kwargs = mock_resolve.call_args
+    assert resolve_args == ("888",)
+    assert resolve_kwargs.get("city") == "ekaterinburg"
+    assert isinstance(resolve_kwargs.get("config"), RealScraperConfig)
     assert result.resolved_slug == 1
     assert result.succeeded == 1
     assert result.rows_inserted == 1
@@ -328,7 +345,7 @@ async def test_sweep_failed_resolve_counts():
     )
 
     with patch(
-        "app.services.scrapers.yandex_newbuilding.resolve_yandex_jk_slug",
+        "scraper_kit.providers.yandex.newbuilding.resolve_yandex_jk_slug",
         new_callable=AsyncMock,
         return_value=None,
     ):
@@ -354,7 +371,7 @@ async def test_sweep_failed_fetch_counts():
         ]
     )
 
-    with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
+    with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper:
         scraper_instance = MagicMock()
         scraper_instance.fetch_jk = AsyncMock(return_value=None)
         mock_scraper.return_value = scraper_instance
@@ -382,7 +399,7 @@ async def test_sweep_idempotent_skip_already_enriched():
     # Pre-populate enrichment
     db.enrichment["555"] = {"ext_id": "555", "name": "Already enriched"}
 
-    with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
+    with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper:
         scraper_instance = MagicMock()
         scraper_instance.fetch_jk = AsyncMock(return_value=None)
         mock_scraper.return_value = scraper_instance
@@ -433,7 +450,7 @@ async def test_sweep_slug_present_but_ext_id_null_skips_fetch():
         ]
     )
 
-    with patch("app.services.scrapers.yandex_newbuilding.YandexNewbuildingScraper") as mock_scraper:
+    with patch("scraper_kit.providers.yandex.newbuilding.YandexNewbuildingScraper") as mock_scraper:
         scraper_instance = MagicMock()
         scraper_instance.fetch_jk = AsyncMock()
         mock_scraper.return_value = scraper_instance