diff --git a/tradein-mvp/backend/app/services/cian_price_history.py b/tradein-mvp/backend/app/services/cian_price_history.py index ec43af46..9069cdb8 100644 --- a/tradein-mvp/backend/app/services/cian_price_history.py +++ b/tradein-mvp/backend/app/services/cian_price_history.py @@ -15,11 +15,17 @@ import logging import time from dataclasses import dataclass, field +# #2306: fetch_detail/save_detail_enrichment migrated to scraper_kit (golden-parity +# with legacy app.services.scrapers.cian_detail, см. tests/test_scraper_kit_cian_golden_parity.py +# + tests/test_scraper_kit_pricehistory_session_parity.py). RealScraperConfig — тот же +# read-only адаптер над settings, что и остальные kit-инжекции (#2131) — сохраняет +# proxy-поведение (config.cian_proxy_url) идентичным прежнему прямому импорту settings. +from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment from sqlalchemy import text from sqlalchemy.orm import Session +from app.services.scraper_adapters import RealScraperConfig from app.services.scraper_settings import get_scraper_delay -from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment logger = logging.getLogger(__name__) @@ -99,7 +105,9 @@ async def backfill_cian_price_history( url: str = row["source_url"] try: - enrichment = await fetch_detail(url) + # config= обязателен — kit fetch_detail без него не читает cian_proxy_url + # (direct connection), а без прокси datacenter-IP блокируется Cian (#806). + enrichment = await fetch_detail(url, config=RealScraperConfig()) except Exception as exc: logger.warning( "cian_price_history: fetch failed listing_id=%s url=%s: %s", diff --git a/tradein-mvp/backend/app/services/cian_session.py b/tradein-mvp/backend/app/services/cian_session.py index 89324edc..9768b41a 100644 --- a/tradein-mvp/backend/app/services/cian_session.py +++ b/tradein-mvp/backend/app/services/cian_session.py @@ -11,11 +11,15 @@ import logging from typing import Any from curl_cffi.requests import AsyncSession + +# #2306: extract_state migrated to scraper_kit (byte-identical to legacy +# app.services.scrapers.cian_state_parser, см. tests/test_scraper_kit_cian_golden_parity.py +# + tests/test_scraper_kit_pricehistory_session_parity.py). +from scraper_kit.cian_state_parser import extract_state from sqlalchemy import text from sqlalchemy.orm import Session from app.core.config import settings -from app.services.scrapers.cian_state_parser import extract_state logger = logging.getLogger(__name__) diff --git a/tradein-mvp/backend/app/services/yandex_price_history.py b/tradein-mvp/backend/app/services/yandex_price_history.py index 7665aad2..5b5a7ab1 100644 --- a/tradein-mvp/backend/app/services/yandex_price_history.py +++ b/tradein-mvp/backend/app/services/yandex_price_history.py @@ -26,11 +26,15 @@ from __future__ import annotations import logging from datetime import UTC, datetime, timedelta +# #2306: ScrapedLot migrated to scraper_kit (byte-identical field/method surface to +# legacy app.services.scrapers.base, см. tests/test_scraper_kit_pricehistory_session_parity.py). +# record_yandex_price_history reads lot.* via duck-typing only — no isinstance checks — +# так что kit- и legacy-скрапперы (оба конструируют ScrapedLot по (source, source_id, +# price_rub, price_previous_rub, price_trend)) остаются взаимозаменяемы для этой функции. +from scraper_kit.base import ScrapedLot from sqlalchemy import text from sqlalchemy.orm import Session -from app.services.scrapers.base import ScrapedLot - logger = logging.getLogger(__name__) _SOURCE = "yandex" diff --git a/tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py new file mode 100644 index 00000000..32039510 --- /dev/null +++ b/tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py @@ -0,0 +1,214 @@ +"""Parity tests for issue #2306 (Group B of scraper_kit migration epic #2277). + +Migrates the INTERNAL legacy-scraper imports of three product-side files to their +`scraper_kit` equivalents (public API of the three files is unchanged — no caller +elsewhere in the codebase needed updating, see vault audit +`Scraper_Kit_Legacy_Dependency_Audit_0703`): + + - app/services/cian_price_history.py → fetch_detail/save_detail_enrichment now from + `scraper_kit.providers.cian.detail` (was `app.services.scrapers.cian_detail`). + - app/services/cian_session.py → extract_state now from + `scraper_kit.cian_state_parser` (was `app.services.scrapers.cian_state_parser`). + - app/services/yandex_price_history.py → ScrapedLot now from `scraper_kit.base` + (was `app.services.scrapers.base`). + +This file proves the DELTA introduced by the migration specifically — general +cian-provider / ScrapedLot golden-parity is already covered by +`tests/test_scraper_kit_cian_golden_parity.py` and +`tests/test_scraper_kit_base_golden_parity.py`. Uses `tests/support/parity.assert_parity` +per the epic's parity-gate convention (issue #2304). +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import json +import os +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +# Старый app.services.scrapers.cian* / app.core.config.settings требует DATABASE_URL +# при импорте (тот же паттерн, что в test_scraper_kit_cian_golden_parity.py). +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +# Kit — то, на что теперь ссылаются мигрированные файлы (#2306). +from scraper_kit.base import ScrapedLot as KitScrapedLot +from scraper_kit.cian_state_parser import extract_state as kit_extract_state +from scraper_kit.providers.cian.detail import fetch_detail as kit_fetch_detail + +from app.core.config import settings +from app.services.scraper_adapters import RealScraperConfig + +# Legacy — эталон сравнения (эти модули НЕ меняются в #2306). +from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot +from app.services.scrapers.cian_detail import fetch_detail as legacy_fetch_detail +from app.services.scrapers.cian_state_parser import extract_state as legacy_extract_state +from tests.support.parity import assert_parity + +# ── extract_state parity (cian_session.py dependency) ───────────────────────── + + +def _auth_state_html() -> str: + state = {"user": {"isAuthenticated": True, "userId": 42}} + state_json = json.dumps(state, ensure_ascii=False) + return ( + "" + ) + + +def test_extract_state_parity_on_match() -> None: + """cian_session.py._classify_verify_response reads this exact (mfe, key) pair.""" + html = _auth_state_html() + assert_parity( + legacy_fn=legacy_extract_state, + kit_fn=kit_extract_state, + fixtures=[(html, "header-frontend", "initialState")], + ) + + +def test_extract_state_parity_on_miss() -> None: + """Missing mfe/key → both return None identically (not just on the happy path).""" + html = _auth_state_html() + assert_parity( + legacy_fn=legacy_extract_state, + kit_fn=kit_extract_state, + fixtures=[(html, "frontend-offer-card", "defaultState")], + ) + + +# ── ScrapedLot parity (yandex_price_history.py dependency) ──────────────────── + + +def _lot_kwargs() -> dict[str, Any]: + return { + "source": "yandex", + "source_url": "https://realty.yandex.ru/offer/123/", + "source_id": "123", + "price_rub": 5_000_000, + "price_previous_rub": 5_200_000, + "price_trend": "DECREASED", + } + + +def test_scraped_lot_model_dump_parity() -> None: + """record_yandex_price_history duck-types lot.source/.source_id/.price_rub/... + + Legacy- и kit-ScrapedLot должны давать идентичный model_dump() на одном входе — + доказывает, что смена типа импорта в yandex_price_history.py не меняет runtime- + поведение (функция принимает оба класса взаимозаменяемо). + """ + kwargs = _lot_kwargs() + assert_parity( + legacy_fn=lambda kw: LegacyScrapedLot(**kw).model_dump(), + kit_fn=lambda kw: KitScrapedLot(**kw).model_dump(), + fixtures=[kwargs], + ) + + +# ── fetch_detail / DetailEnrichment parity (cian_price_history.py dependency) ── + + +def _detail_html() -> str: + default_state = { + "offerData": { + "offer": { + "cianId": 555111, + "priceChanges": [ + {"changeTime": "2026-05-01T00:00:00Z", "price": 6100000, "diffPercent": -1.2} + ], + }, + "stats": {}, + } + } + ds_json = json.dumps(default_state, ensure_ascii=False) + return ( + "" + ) + + +def _mock_session(html: str) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.text = html + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + return session + + +def _run_legacy_fetch(url: str, html: str) -> Any: + return dataclasses.asdict(asyncio.run(legacy_fetch_detail(url, session=_mock_session(html)))) + + +def _run_kit_fetch(url: str, html: str) -> Any: + # Точно тот вызов, что теперь делает cian_price_history.backfill_cian_price_history: + # config=RealScraperConfig() добавлен по миграции (#2306); session= общий mock здесь + # только чтобы избежать сетевого I/O — сам config не участвует в session-пути, так + # что этот тест доказывает: инжекция config= не меняет результат парсинга. + result = asyncio.run( + kit_fetch_detail(url, config=RealScraperConfig(), session=_mock_session(html)) + ) + return dataclasses.asdict(result) + + +def test_fetch_detail_with_config_injection_parity() -> None: + html = _detail_html() + url = "https://ekb.cian.ru/sale/flat/555111/" + assert_parity( + legacy_fn=_run_legacy_fetch, + kit_fn=_run_kit_fetch, + fixtures=[(url, html)], + ) + + +def test_fetch_detail_own_session_proxy_wiring_parity(monkeypatch: Any) -> None: + """cian_price_history calls fetch_detail WITHOUT session= (own-session curl_cffi path). + + Kit fetch_detail without config= silently drops the proxy (direct connection) — a + real behaviour regression risk (#806: datacenter-IP gets banned by Cian without a + proxy). This proves config=RealScraperConfig() (our actual call site) constructs + AsyncSession(proxies=...) identically to the legacy direct-settings path. + """ + monkeypatch.setattr(settings, "cian_proxy_url_env", "http://test-proxy.local:8080") + + captured: dict[str, dict[str, Any]] = {} + + def _make_fake_session(name: str) -> Any: + class _FakeSession: + def __init__(self, **kwargs: Any) -> None: + captured[name] = kwargs + + async def get(self, *_args: Any, **_kwargs: Any) -> Any: + response = MagicMock() + response.status_code = 200 + response.text = _detail_html() + return response + + async def close(self) -> None: + pass + + return _FakeSession + + url = "https://ekb.cian.ru/sale/flat/1/" + + with patch("app.services.scrapers.cian_detail.AsyncSession", _make_fake_session("legacy")): + asyncio.run(legacy_fetch_detail(url)) + + with patch("scraper_kit.providers.cian.detail.AsyncSession", _make_fake_session("kit")): + asyncio.run(kit_fetch_detail(url, config=RealScraperConfig())) + + assert captured["legacy"].get("proxies") == captured["kit"].get("proxies") + assert captured["kit"]["proxies"] == { + "http": "http://test-proxy.local:8080", + "https": "http://test-proxy.local:8080", + }