Merge pull request 'feat(tradein/scraper-kit): migrate cian/yandex price-history & session to kit (#2306)' (#2324) from feat/tradein-migrate-price-history-kit into main

code-reviewer flagged that neither the new parity test nor
test_backfill_wave2.py's 3 cian_price_history tests would catch a
regression if config=RealScraperConfig() were accidentally dropped from
the fetch_detail call in cian_price_history.py (the proxy-footgun fix
from the #2306 migration) — mock_fetch.assert_called_once() doesn't
check kwargs.

Strengthens all 3 tests (saves_changes / skips_no_changes /
handles_fetch_none) to assert isinstance(call_kwargs["config"],
RealScraperConfig). Manually verified the guard actually catches the
regression: temporarily removed config= from cian_price_history.py:110,
confirmed all 3 tests fail, restored the fix, confirmed they pass again.

tradein-mvp/backend/app/services/cian_price_history.py

@@ -15,11 +15,17 @@ import logging
 import time
 from dataclasses import dataclass, field
 
+# #2306: fetch_detail/save_detail_enrichment migrated to scraper_kit (golden-parity
+# with legacy app.services.scrapers.cian_detail, см. tests/test_scraper_kit_cian_golden_parity.py
+# + tests/test_scraper_kit_pricehistory_session_parity.py). RealScraperConfig — тот же
+# read-only адаптер над settings, что и остальные kit-инжекции (#2131) — сохраняет
+# proxy-поведение (config.cian_proxy_url) идентичным прежнему прямому импорту settings.
+from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
+from app.services.scraper_adapters import RealScraperConfig
 from app.services.scraper_settings import get_scraper_delay
-from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment
 
 logger = logging.getLogger(__name__)
 
@@ -99,7 +105,9 @@ async def backfill_cian_price_history(
         url: str = row["source_url"]
 
         try:
-            enrichment = await fetch_detail(url)
+            # config= обязателен — kit fetch_detail без него не читает cian_proxy_url
+            # (direct connection), а без прокси datacenter-IP блокируется Cian (#806).
+            enrichment = await fetch_detail(url, config=RealScraperConfig())
         except Exception as exc:
             logger.warning(
                 "cian_price_history: fetch failed listing_id=%s url=%s: %s",

tradein-mvp/backend/app/services/cian_session.py

@@ -11,11 +11,15 @@ import logging
 from typing import Any
 
 from curl_cffi.requests import AsyncSession
+
+# #2306: extract_state migrated to scraper_kit (byte-identical to legacy
+# app.services.scrapers.cian_state_parser, см. tests/test_scraper_kit_cian_golden_parity.py
+# + tests/test_scraper_kit_pricehistory_session_parity.py).
+from scraper_kit.cian_state_parser import extract_state
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from app.core.config import settings
-from app.services.scrapers.cian_state_parser import extract_state
 
 logger = logging.getLogger(__name__)
 

tradein-mvp/backend/app/services/yandex_price_history.py

@@ -26,11 +26,15 @@ from __future__ import annotations
 import logging
 from datetime import UTC, datetime, timedelta
 
+# #2306: ScrapedLot migrated to scraper_kit (byte-identical field/method surface to
+# legacy app.services.scrapers.base, см. tests/test_scraper_kit_pricehistory_session_parity.py).
+# record_yandex_price_history reads lot.* via duck-typing only — no isinstance checks —
+# так что kit- и legacy-скрапперы (оба конструируют ScrapedLot по (source, source_id,
+# price_rub, price_previous_rub, price_trend)) остаются взаимозаменяемы для этой функции.
+from scraper_kit.base import ScrapedLot
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
-from app.services.scrapers.base import ScrapedLot
-
 logger = logging.getLogger(__name__)
 
 _SOURCE = "yandex"

tradein-mvp/backend/tests/test_backfill_wave2.py

@@ -235,6 +235,7 @@ async def test_backfill_yandex_addresses_error_on_non_200():
 async def test_cian_price_history_saves_changes():
     """Happy path: fetch returns DetailEnrichment with price_changes → saved."""
     from app.services.cian_price_history import backfill_cian_price_history
+    from app.services.scraper_adapters import RealScraperConfig
     from app.services.scrapers.cian_detail import DetailEnrichment
 
     enrichment = DetailEnrichment(
@@ -277,6 +278,12 @@ async def test_cian_price_history_saves_changes():
     assert result.saved == 2  # 2 price_changes
     assert result.errors == 0
     mock_fetch.assert_called_once()
+    # #2306 regression guard: kit fetch_detail silently drops the Cian proxy
+    # (direct connection, #806 ban risk) unless config=RealScraperConfig() is
+    # passed at the call site — assert_called_once() alone wouldn't catch someone
+    # dropping that kwarg later.
+    _, call_kwargs = mock_fetch.call_args
+    assert isinstance(call_kwargs.get("config"), RealScraperConfig)
     mock_save.assert_called_once()
 
 
@@ -284,6 +291,7 @@ async def test_cian_price_history_saves_changes():
 async def test_cian_price_history_skips_no_changes():
     """Fetch succeeds but no price_changes → skipped (not error)."""
     from app.services.cian_price_history import backfill_cian_price_history
+    from app.services.scraper_adapters import RealScraperConfig
     from app.services.scrapers.cian_detail import DetailEnrichment
 
     enrichment = DetailEnrichment(cian_id=111, price_changes=[])
@@ -296,7 +304,7 @@ async def test_cian_price_history_skips_no_changes():
             "app.services.cian_price_history.fetch_detail",
             new_callable=AsyncMock,
             return_value=enrichment,
-        ),
+        ) as mock_fetch,
         patch("app.services.cian_price_history.save_detail_enrichment") as mock_save,
         patch("app.services.cian_price_history.get_scraper_delay", return_value=0.0),
     ):
@@ -309,12 +317,16 @@ async def test_cian_price_history_skips_no_changes():
     assert result.skipped == 1
     assert result.saved == 0
     mock_save.assert_not_called()
+    # #2306 regression guard — see test_cian_price_history_saves_changes.
+    _, call_kwargs = mock_fetch.call_args
+    assert isinstance(call_kwargs.get("config"), RealScraperConfig)
 
 
 @pytest.mark.asyncio
 async def test_cian_price_history_handles_fetch_none():
     """fetch_detail returns None → errors+1, no crash."""
     from app.services.cian_price_history import backfill_cian_price_history
+    from app.services.scraper_adapters import RealScraperConfig
 
     mock_db = MagicMock()
     rows = [{"id": 30, "source_url": "https://ekb.cian.ru/sale/flat/999/"}]
@@ -324,7 +336,7 @@ async def test_cian_price_history_handles_fetch_none():
             "app.services.cian_price_history.fetch_detail",
             new_callable=AsyncMock,
             return_value=None,
-        ),
+        ) as mock_fetch,
         patch("app.services.cian_price_history.get_scraper_delay", return_value=0.0),
     ):
         mock_mappings = MagicMock()
@@ -335,6 +347,9 @@ async def test_cian_price_history_handles_fetch_none():
 
     assert result.errors == 1
     assert result.saved == 0
+    # #2306 regression guard — see test_cian_price_history_saves_changes.
+    _, call_kwargs = mock_fetch.call_args
+    assert isinstance(call_kwargs.get("config"), RealScraperConfig)
 
 
 # ============================================================================

tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py

@@ -0,0 +1,214 @@
+"""Parity tests for issue #2306 (Group B of scraper_kit migration epic #2277).
+
+Migrates the INTERNAL legacy-scraper imports of three product-side files to their
+`scraper_kit` equivalents (public API of the three files is unchanged — no caller
+elsewhere in the codebase needed updating, see vault audit
+`Scraper_Kit_Legacy_Dependency_Audit_0703`):
+
+  - app/services/cian_price_history.py  → fetch_detail/save_detail_enrichment now from
+    `scraper_kit.providers.cian.detail` (was `app.services.scrapers.cian_detail`).
+  - app/services/cian_session.py        → extract_state now from
+    `scraper_kit.cian_state_parser` (was `app.services.scrapers.cian_state_parser`).
+  - app/services/yandex_price_history.py → ScrapedLot now from `scraper_kit.base`
+    (was `app.services.scrapers.base`).
+
+This file proves the DELTA introduced by the migration specifically — general
+cian-provider / ScrapedLot golden-parity is already covered by
+`tests/test_scraper_kit_cian_golden_parity.py` and
+`tests/test_scraper_kit_base_golden_parity.py`. Uses `tests/support/parity.assert_parity`
+per the epic's parity-gate convention (issue #2304).
+"""
+
+from __future__ import annotations
+
+import asyncio
+import dataclasses
+import json
+import os
+from typing import Any
+from unittest.mock import AsyncMock, MagicMock, patch
+
+# Старый app.services.scrapers.cian* / app.core.config.settings требует DATABASE_URL
+# при импорте (тот же паттерн, что в test_scraper_kit_cian_golden_parity.py).
+os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
+
+# Kit — то, на что теперь ссылаются мигрированные файлы (#2306).
+from scraper_kit.base import ScrapedLot as KitScrapedLot
+from scraper_kit.cian_state_parser import extract_state as kit_extract_state
+from scraper_kit.providers.cian.detail import fetch_detail as kit_fetch_detail
+
+from app.core.config import settings
+from app.services.scraper_adapters import RealScraperConfig
+
+# Legacy — эталон сравнения (эти модули НЕ меняются в #2306).
+from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot
+from app.services.scrapers.cian_detail import fetch_detail as legacy_fetch_detail
+from app.services.scrapers.cian_state_parser import extract_state as legacy_extract_state
+from tests.support.parity import assert_parity
+
+# ── extract_state parity (cian_session.py dependency) ─────────────────────────
+
+
+def _auth_state_html() -> str:
+    state = {"user": {"isAuthenticated": True, "userId": 42}}
+    state_json = json.dumps(state, ensure_ascii=False)
+    return (
+        "<html><body><script>\n"
+        "window._cianConfig['header-frontend'] = window._cianConfig['header-frontend'] || [];\n"
+        "window._cianConfig['header-frontend'].push({"
+        f"key: 'initialState', value: {state_json}, priority: 10, filter: null"
+        "});\n</script></body></html>"
+    )
+
+
+def test_extract_state_parity_on_match() -> None:
+    """cian_session.py._classify_verify_response reads this exact (mfe, key) pair."""
+    html = _auth_state_html()
+    assert_parity(
+        legacy_fn=legacy_extract_state,
+        kit_fn=kit_extract_state,
+        fixtures=[(html, "header-frontend", "initialState")],
+    )
+
+
+def test_extract_state_parity_on_miss() -> None:
+    """Missing mfe/key → both return None identically (not just on the happy path)."""
+    html = _auth_state_html()
+    assert_parity(
+        legacy_fn=legacy_extract_state,
+        kit_fn=kit_extract_state,
+        fixtures=[(html, "frontend-offer-card", "defaultState")],
+    )
+
+
+# ── ScrapedLot parity (yandex_price_history.py dependency) ────────────────────
+
+
+def _lot_kwargs() -> dict[str, Any]:
+    return {
+        "source": "yandex",
+        "source_url": "https://realty.yandex.ru/offer/123/",
+        "source_id": "123",
+        "price_rub": 5_000_000,
+        "price_previous_rub": 5_200_000,
+        "price_trend": "DECREASED",
+    }
+
+
+def test_scraped_lot_model_dump_parity() -> None:
+    """record_yandex_price_history duck-types lot.source/.source_id/.price_rub/...
+
+    Legacy- и kit-ScrapedLot должны давать идентичный model_dump() на одном входе —
+    доказывает, что смена типа импорта в yandex_price_history.py не меняет runtime-
+    поведение (функция принимает оба класса взаимозаменяемо).
+    """
+    kwargs = _lot_kwargs()
+    assert_parity(
+        legacy_fn=lambda kw: LegacyScrapedLot(**kw).model_dump(),
+        kit_fn=lambda kw: KitScrapedLot(**kw).model_dump(),
+        fixtures=[kwargs],
+    )
+
+
+# ── fetch_detail / DetailEnrichment parity (cian_price_history.py dependency) ──
+
+
+def _detail_html() -> str:
+    default_state = {
+        "offerData": {
+            "offer": {
+                "cianId": 555111,
+                "priceChanges": [
+                    {"changeTime": "2026-05-01T00:00:00Z", "price": 6100000, "diffPercent": -1.2}
+                ],
+            },
+            "stats": {},
+        }
+    }
+    ds_json = json.dumps(default_state, ensure_ascii=False)
+    return (
+        "<html><body><script>\n"
+        "window._cianConfig['frontend-offer-card'] = "
+        "window._cianConfig['frontend-offer-card'] || [];\n"
+        "window._cianConfig['frontend-offer-card'].push({"
+        f"key: 'defaultState', value: {ds_json}, priority: 10, filter: null"
+        "});\n</script></body></html>"
+    )
+
+
+def _mock_session(html: str) -> MagicMock:
+    response = MagicMock()
+    response.status_code = 200
+    response.text = html
+    session = MagicMock()
+    session.get = AsyncMock(return_value=response)
+    session.close = AsyncMock()
+    return session
+
+
+def _run_legacy_fetch(url: str, html: str) -> Any:
+    return dataclasses.asdict(asyncio.run(legacy_fetch_detail(url, session=_mock_session(html))))
+
+
+def _run_kit_fetch(url: str, html: str) -> Any:
+    # Точно тот вызов, что теперь делает cian_price_history.backfill_cian_price_history:
+    # config=RealScraperConfig() добавлен по миграции (#2306); session= общий mock здесь
+    # только чтобы избежать сетевого I/O — сам config не участвует в session-пути, так
+    # что этот тест доказывает: инжекция config= не меняет результат парсинга.
+    result = asyncio.run(
+        kit_fetch_detail(url, config=RealScraperConfig(), session=_mock_session(html))
+    )
+    return dataclasses.asdict(result)
+
+
+def test_fetch_detail_with_config_injection_parity() -> None:
+    html = _detail_html()
+    url = "https://ekb.cian.ru/sale/flat/555111/"
+    assert_parity(
+        legacy_fn=_run_legacy_fetch,
+        kit_fn=_run_kit_fetch,
+        fixtures=[(url, html)],
+    )
+
+
+def test_fetch_detail_own_session_proxy_wiring_parity(monkeypatch: Any) -> None:
+    """cian_price_history calls fetch_detail WITHOUT session= (own-session curl_cffi path).
+
+    Kit fetch_detail without config= silently drops the proxy (direct connection) — a
+    real behaviour regression risk (#806: datacenter-IP gets banned by Cian without a
+    proxy). This proves config=RealScraperConfig() (our actual call site) constructs
+    AsyncSession(proxies=...) identically to the legacy direct-settings path.
+    """
+    monkeypatch.setattr(settings, "cian_proxy_url_env", "http://test-proxy.local:8080")
+
+    captured: dict[str, dict[str, Any]] = {}
+
+    def _make_fake_session(name: str) -> Any:
+        class _FakeSession:
+            def __init__(self, **kwargs: Any) -> None:
+                captured[name] = kwargs
+
+            async def get(self, *_args: Any, **_kwargs: Any) -> Any:
+                response = MagicMock()
+                response.status_code = 200
+                response.text = _detail_html()
+                return response
+
+            async def close(self) -> None:
+                pass
+
+        return _FakeSession
+
+    url = "https://ekb.cian.ru/sale/flat/1/"
+
+    with patch("app.services.scrapers.cian_detail.AsyncSession", _make_fake_session("legacy")):
+        asyncio.run(legacy_fetch_detail(url))
+
+    with patch("scraper_kit.providers.cian.detail.AsyncSession", _make_fake_session("kit")):
+        asyncio.run(kit_fetch_detail(url, config=RealScraperConfig()))
+
+    assert captured["legacy"].get("proxies") == captured["kit"].get("proxies")
+    assert captured["kit"]["proxies"] == {
+        "http": "http://test-proxy.local:8080",
+        "https": "http://test-proxy.local:8080",
+    }