From f0c7ac49720e78ec76b89b2ef438a39be9ceedc9 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 15:17:27 +0300 Subject: [PATCH] =?UTF-8?q?refactor(tradein/tasks):=20cian=5Fhistory=5Fbac?= =?UTF-8?q?kfill=20valuation+newbuilding=20=D0=BD=D0=B0=20kit=20+=20config?= =?UTF-8?q?=3D=20(#2397=20Part=20D2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../app/tasks/cian_history_backfill.py | 20 ++--- ...scraper_kit_group_c_backfill_kit_parity.py | 85 +++++++++++++++++-- 2 files changed, 88 insertions(+), 17 deletions(-) diff --git a/tradein-mvp/backend/app/tasks/cian_history_backfill.py b/tradein-mvp/backend/app/tasks/cian_history_backfill.py index 414afc98..2292bb37 100644 --- a/tradein-mvp/backend/app/tasks/cian_history_backfill.py +++ b/tradein-mvp/backend/app/tasks/cian_history_backfill.py @@ -29,17 +29,14 @@ from dataclasses import dataclass, field from scraper_kit.browser_fetcher import BrowserFetcher from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment +from scraper_kit.providers.cian.valuation import estimate_via_cian_valuation from sqlalchemy import text from sqlalchemy.orm import Session from app.core.config import settings +from app.services.scraper_adapters import RealScraperConfig from app.services.scraper_settings import get_scraper_delay -# cian_valuation остаётся legacy — валюационные зависимости (avito_imv/ -# cian_valuation/yandex_valuation) выделены в отдельный, более рискованный -# issue #2308 (см. epic #2277, issue #2310 Group C), не в этой миграции. -from app.services.scrapers.cian_valuation import estimate_via_cian_valuation - logger = logging.getLogger(__name__) @@ -162,12 +159,10 @@ async def backfill_cian_history( # ── 2. Houses: missing houses_price_dynamics ────────────────────────────── if do_houses: - # cian_newbuilding остаётся legacy (issue #2310, NOT migrated): kit's - # scraper_kit.providers.cian.newbuilding.fetch_newbuilding() constructs - # BrowserFetcher(source="cian") WITHOUT the now-mandatory endpoint= kwarg - # → TypeError on every call, no caller-side fix possible (issue #2322, - # still open — verified against provider source, not just issue status). - from app.services.scrapers.cian_newbuilding import ( + # Kit's fetch_newbuilding() now accepts config= (issue #2322 fixed) — pass + # RealScraperConfig() at the call site below so BrowserFetcher gets a real + # endpoint instead of degrading to endpoint=None (#2397 Part D2). + from scraper_kit.providers.cian.newbuilding import ( fetch_newbuilding, save_newbuilding_enrichment, ) @@ -201,7 +196,7 @@ async def backfill_cian_history( enrichment = None try: - enrichment = await fetch_newbuilding(zhk_url) + enrichment = await fetch_newbuilding(zhk_url, config=RealScraperConfig()) except Exception as exc: logger.warning( "cian_newbuilding fetch failed for house_id=%s url=%s: %s", @@ -283,6 +278,7 @@ async def backfill_cian_history( try: cval = await estimate_via_cian_valuation( db, + config=RealScraperConfig(), address=row["address"], total_area=float(row["area_m2"]), rooms_count=int(row["rooms"]), diff --git a/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py index 6fd9acf0..98d5e3f8 100644 --- a/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py +++ b/tradein-mvp/backend/tests/test_scraper_kit_group_c_backfill_kit_parity.py @@ -21,11 +21,13 @@ Migrates legacy scraper imports of these files to their `scraper_kit` equivalent before the sticky MGTS-proxy egress on the warm-batch path (avito_detail_backfill_use_curl=True, the prod default) can move to kit. - app/tasks/cian_history_backfill.py → fetch_detail/save_detail_enrichment - (listings block) + BrowserFetcher now from `scraper_kit.*`. The houses block - (cian_newbuilding.fetch_newbuilding/save_newbuilding_enrichment) and the - valuations block (cian_valuation.estimate_via_cian_valuation) stay legacy — - issue #2322 (newbuilding BrowserFetcher missing endpoint=) resp. issue #2308 - (valuation dependencies), both out of scope here. + (listings block), fetch_newbuilding/save_newbuilding_enrichment (houses block), + and estimate_via_cian_valuation (valuations block) all now from `scraper_kit.*`. + Both the houses and valuations call sites pass `config=RealScraperConfig()` + (mandatory kwarg on the kit functions) — see + `test_cian_history_backfill_houses_fetch_newbuilding_gets_config_endpoint` and + `test_cian_history_backfill_valuations_estimate_gets_config` below (#2397 Part D2; + houses block was previously blocked on issue #2322, now fixed). - app/tasks/ekb_geoportal_ingest.py → EkbGeoportalClient/TopoBuilding now from `scraper_kit.providers.ekb_geoportal.client` (already proven byte- identical by tests/scrapers/test_ekb_geoportal_client_kit_parity.py, #2307). @@ -238,6 +240,79 @@ async def test_cian_history_backfill_browser_fetcher_uses_settings_endpoint() -> assert captured == {"source": "cian", "endpoint": settings.browser_http_endpoint} +# ── config= threading — houses / valuations blocks (#2397 Part D2) ─────────────── + + +async def test_cian_history_backfill_houses_fetch_newbuilding_gets_config_endpoint() -> None: + """Houses block's fetch_newbuilding() call must pass config=RealScraperConfig() so + kit's BrowserFetcher gets a real browser_http_endpoint (issue #2322, now fixed) — + without it fetch_newbuilding() degrades to endpoint=None instead of raising, which + would silently no-op the nightly houses_price_dynamics backfill in prod.""" + from app.tasks import cian_history_backfill + + captured: dict[str, Any] = {} + + async def _fake_fetch_newbuilding(zhk_url: str, *, config: Any = None) -> None: + captured["zhk_url"] = zhk_url + captured["endpoint"] = config.browser_http_endpoint if config is not None else None + + db = MagicMock() + db.execute.return_value.mappings.return_value.all.return_value = [ + {"id": 1, "cian_zhk_url": "https://zhk-test-ekb-i.cian.ru/"} + ] + + with ( + patch( + "scraper_kit.providers.cian.newbuilding.fetch_newbuilding", + _fake_fetch_newbuilding, + ), + patch("asyncio.sleep", new_callable=AsyncMock), + ): + await cian_history_backfill.backfill_cian_history( + db, do_listings=False, do_houses=True, do_valuations=False + ) + + assert captured["zhk_url"] == "https://zhk-test-ekb-i.cian.ru/" + assert captured["endpoint"] == settings.browser_http_endpoint + + +async def test_cian_history_backfill_valuations_estimate_gets_config() -> None: + """Valuations block's estimate_via_cian_valuation() call must pass a non-None + config=RealScraperConfig() — kit's function requires config as a mandatory + keyword-only arg (load_session(db, config=config) and the price sanity gate both + read it); omitting it is a TypeError at call time, not a silent degrade.""" + from app.tasks import cian_history_backfill + + captured: dict[str, Any] = {} + + async def _fake_estimate(db_arg: Any, *, config: Any = None, **kwargs: Any) -> None: + captured["config"] = config + captured["kwargs"] = kwargs + + db = MagicMock() + db.execute.return_value.mappings.return_value.all.return_value = [ + { + "id": 1, + "address": "ЕКБ, ул. Тестовая, 1", + "area_m2": 45.0, + "rooms": 2, + "floor": 3, + "total_floors": 9, + } + ] + + with ( + patch.object(cian_history_backfill, "estimate_via_cian_valuation", _fake_estimate), + patch("asyncio.sleep", new_callable=AsyncMock), + ): + await cian_history_backfill.backfill_cian_history( + db, do_listings=False, do_houses=False, do_valuations=True + ) + + assert captured["config"] is not None + assert isinstance(captured["config"], RealScraperConfig) + + # ── ekb_geoportal_ingest.py: task module resolves the kit client import ──────────