refactor(tradein/tasks): cian_history_backfill valuation+newbuilding на kit + config= (#2397 Part D2)

tradein-mvp/backend/app/tasks/cian_history_backfill.py

@@ -29,17 +29,14 @@ from dataclasses import dataclass, field
 
 from scraper_kit.browser_fetcher import BrowserFetcher
 from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment
+from scraper_kit.providers.cian.valuation import estimate_via_cian_valuation
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from app.core.config import settings
+from app.services.scraper_adapters import RealScraperConfig
 from app.services.scraper_settings import get_scraper_delay
 
-# cian_valuation остаётся legacy — валюационные зависимости (avito_imv/
-# cian_valuation/yandex_valuation) выделены в отдельный, более рискованный
-# issue #2308 (см. epic #2277, issue #2310 Group C), не в этой миграции.
-from app.services.scrapers.cian_valuation import estimate_via_cian_valuation
-
 logger = logging.getLogger(__name__)
 
 
@@ -162,12 +159,10 @@ async def backfill_cian_history(
 
     # ── 2. Houses: missing houses_price_dynamics ──────────────────────────────
     if do_houses:
-        # cian_newbuilding остаётся legacy (issue #2310, NOT migrated): kit's
-        # scraper_kit.providers.cian.newbuilding.fetch_newbuilding() constructs
-        # BrowserFetcher(source="cian") WITHOUT the now-mandatory endpoint= kwarg
-        # → TypeError on every call, no caller-side fix possible (issue #2322,
-        # still open — verified against provider source, not just issue status).
-        from app.services.scrapers.cian_newbuilding import (
+        # Kit's fetch_newbuilding() now accepts config= (issue #2322 fixed) — pass
+        # RealScraperConfig() at the call site below so BrowserFetcher gets a real
+        # endpoint instead of degrading to endpoint=None (#2397 Part D2).
+        from scraper_kit.providers.cian.newbuilding import (
             fetch_newbuilding,
             save_newbuilding_enrichment,
         )
@@ -201,7 +196,7 @@ async def backfill_cian_history(
 
                 enrichment = None
                 try:
-                    enrichment = await fetch_newbuilding(zhk_url)
+                    enrichment = await fetch_newbuilding(zhk_url, config=RealScraperConfig())
                 except Exception as exc:
                     logger.warning(
                         "cian_newbuilding fetch failed for house_id=%s url=%s: %s",
@@ -283,6 +278,7 @@ async def backfill_cian_history(
                 try:
                     cval = await estimate_via_cian_valuation(
                         db,
+                        config=RealScraperConfig(),
                         address=row["address"],
                         total_area=float(row["area_m2"]),
                         rooms_count=int(row["rooms"]),

tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py

@@ -21,11 +21,13 @@ Migrates legacy scraper imports of these files to their `scraper_kit` equivalent
     before the sticky MGTS-proxy egress on the warm-batch path
     (avito_detail_backfill_use_curl=True, the prod default) can move to kit.
   - app/tasks/cian_history_backfill.py    → fetch_detail/save_detail_enrichment
-    (listings block) + BrowserFetcher now from `scraper_kit.*`. The houses block
-    (cian_newbuilding.fetch_newbuilding/save_newbuilding_enrichment) and the
-    valuations block (cian_valuation.estimate_via_cian_valuation) stay legacy —
-    issue #2322 (newbuilding BrowserFetcher missing endpoint=) resp. issue #2308
-    (valuation dependencies), both out of scope here.
+    (listings block), fetch_newbuilding/save_newbuilding_enrichment (houses block),
+    and estimate_via_cian_valuation (valuations block) all now from `scraper_kit.*`.
+    Both the houses and valuations call sites pass `config=RealScraperConfig()`
+    (mandatory kwarg on the kit functions) — see
+    `test_cian_history_backfill_houses_fetch_newbuilding_gets_config_endpoint` and
+    `test_cian_history_backfill_valuations_estimate_gets_config` below (#2397 Part D2;
+    houses block was previously blocked on issue #2322, now fixed).
   - app/tasks/ekb_geoportal_ingest.py     → EkbGeoportalClient/TopoBuilding now
     from `scraper_kit.providers.ekb_geoportal.client` (already proven byte-
     identical by tests/scrapers/test_ekb_geoportal_client_kit_parity.py, #2307).
@@ -238,6 +240,79 @@ async def test_cian_history_backfill_browser_fetcher_uses_settings_endpoint() ->
     assert captured == {"source": "cian", "endpoint": settings.browser_http_endpoint}
 
 
+# ── config= threading — houses / valuations blocks (#2397 Part D2) ───────────────
+
+
+async def test_cian_history_backfill_houses_fetch_newbuilding_gets_config_endpoint() -> None:
+    """Houses block's fetch_newbuilding() call must pass config=RealScraperConfig() so
+    kit's BrowserFetcher gets a real browser_http_endpoint (issue #2322, now fixed) —
+    without it fetch_newbuilding() degrades to endpoint=None instead of raising, which
+    would silently no-op the nightly houses_price_dynamics backfill in prod."""
+    from app.tasks import cian_history_backfill
+
+    captured: dict[str, Any] = {}
+
+    async def _fake_fetch_newbuilding(zhk_url: str, *, config: Any = None) -> None:
+        captured["zhk_url"] = zhk_url
+        captured["endpoint"] = config.browser_http_endpoint if config is not None else None
+
+    db = MagicMock()
+    db.execute.return_value.mappings.return_value.all.return_value = [
+        {"id": 1, "cian_zhk_url": "https://zhk-test-ekb-i.cian.ru/"}
+    ]
+
+    with (
+        patch(
+            "scraper_kit.providers.cian.newbuilding.fetch_newbuilding",
+            _fake_fetch_newbuilding,
+        ),
+        patch("asyncio.sleep", new_callable=AsyncMock),
+    ):
+        await cian_history_backfill.backfill_cian_history(
+            db, do_listings=False, do_houses=True, do_valuations=False
+        )
+
+    assert captured["zhk_url"] == "https://zhk-test-ekb-i.cian.ru/"
+    assert captured["endpoint"] == settings.browser_http_endpoint
+
+
+async def test_cian_history_backfill_valuations_estimate_gets_config() -> None:
+    """Valuations block's estimate_via_cian_valuation() call must pass a non-None
+    config=RealScraperConfig() — kit's function requires config as a mandatory
+    keyword-only arg (load_session(db, config=config) and the price sanity gate both
+    read it); omitting it is a TypeError at call time, not a silent degrade."""
+    from app.tasks import cian_history_backfill
+
+    captured: dict[str, Any] = {}
+
+    async def _fake_estimate(db_arg: Any, *, config: Any = None, **kwargs: Any) -> None:
+        captured["config"] = config
+        captured["kwargs"] = kwargs
+
+    db = MagicMock()
+    db.execute.return_value.mappings.return_value.all.return_value = [
+        {
+            "id": 1,
+            "address": "ЕКБ, ул. Тестовая, 1",
+            "area_m2": 45.0,
+            "rooms": 2,
+            "floor": 3,
+            "total_floors": 9,
+        }
+    ]
+
+    with (
+        patch.object(cian_history_backfill, "estimate_via_cian_valuation", _fake_estimate),
+        patch("asyncio.sleep", new_callable=AsyncMock),
+    ):
+        await cian_history_backfill.backfill_cian_history(
+            db, do_listings=False, do_houses=False, do_valuations=True
+        )
+
+    assert captured["config"] is not None
+    assert isinstance(captured["config"], RealScraperConfig)
+
+
 # ── ekb_geoportal_ingest.py: task module resolves the kit client import ──────────