From db3afa317624af518b21aad832d866ca8aa9548b Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 00:13:07 +0300 Subject: [PATCH 1/2] fix(tradein/scrapers): migrate cian_price_history/cian_session/yandex_price_history imports to scraper_kit (#2306) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Group B of the scraper_kit migration epic (#2277): switch the three files' internal dependency imports from legacy app/services/scrapers/* to their byte/structurally-identical scraper_kit equivalents, proven via the new parity harness (tests/support/parity.assert_parity, #2304). Public API of all three files is unchanged, so no callers needed updating. - cian_price_history.py: fetch_detail/save_detail_enrichment now from scraper_kit.providers.cian.detail. Wires config=RealScraperConfig() at the call site — kit's fetch_detail only reads cian_proxy_url when an explicit config is passed, unlike legacy's always-on settings read; without this the admin-triggered backfill would silently drop its datacenter-IP proxy (#806) after the import swap. - cian_session.py: extract_state now from scraper_kit.cian_state_parser (byte-identical to legacy). - yandex_price_history.py: ScrapedLot now from scraper_kit.base (byte-identical fields/methods; record_yandex_price_history only reads lot.* via duck-typing, no isinstance checks). New tests/test_scraper_kit_pricehistory_session_parity.py proves the migration delta specifically (extract_state, ScrapedLot.model_dump(), fetch_detail+DetailEnrichment with config injection, and the own-session proxy-kwarg wiring). Full backend suite passes (3145 passed, 6 skipped, 1 pre-existing unrelated failure confirmed on unmodified forgejo/main). --- .../app/services/cian_price_history.py | 12 +- .../backend/app/services/cian_session.py | 6 +- .../app/services/yandex_price_history.py | 8 +- ...scraper_kit_pricehistory_session_parity.py | 214 ++++++++++++++++++ 4 files changed, 235 insertions(+), 5 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py diff --git a/tradein-mvp/backend/app/services/cian_price_history.py b/tradein-mvp/backend/app/services/cian_price_history.py index ec43af46..9069cdb8 100644 --- a/tradein-mvp/backend/app/services/cian_price_history.py +++ b/tradein-mvp/backend/app/services/cian_price_history.py @@ -15,11 +15,17 @@ import logging import time from dataclasses import dataclass, field +# #2306: fetch_detail/save_detail_enrichment migrated to scraper_kit (golden-parity +# with legacy app.services.scrapers.cian_detail, см. tests/test_scraper_kit_cian_golden_parity.py +# + tests/test_scraper_kit_pricehistory_session_parity.py). RealScraperConfig — тот же +# read-only адаптер над settings, что и остальные kit-инжекции (#2131) — сохраняет +# proxy-поведение (config.cian_proxy_url) идентичным прежнему прямому импорту settings. +from scraper_kit.providers.cian.detail import fetch_detail, save_detail_enrichment from sqlalchemy import text from sqlalchemy.orm import Session +from app.services.scraper_adapters import RealScraperConfig from app.services.scraper_settings import get_scraper_delay -from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment logger = logging.getLogger(__name__) @@ -99,7 +105,9 @@ async def backfill_cian_price_history( url: str = row["source_url"] try: - enrichment = await fetch_detail(url) + # config= обязателен — kit fetch_detail без него не читает cian_proxy_url + # (direct connection), а без прокси datacenter-IP блокируется Cian (#806). + enrichment = await fetch_detail(url, config=RealScraperConfig()) except Exception as exc: logger.warning( "cian_price_history: fetch failed listing_id=%s url=%s: %s", diff --git a/tradein-mvp/backend/app/services/cian_session.py b/tradein-mvp/backend/app/services/cian_session.py index 89324edc..9768b41a 100644 --- a/tradein-mvp/backend/app/services/cian_session.py +++ b/tradein-mvp/backend/app/services/cian_session.py @@ -11,11 +11,15 @@ import logging from typing import Any from curl_cffi.requests import AsyncSession + +# #2306: extract_state migrated to scraper_kit (byte-identical to legacy +# app.services.scrapers.cian_state_parser, см. tests/test_scraper_kit_cian_golden_parity.py +# + tests/test_scraper_kit_pricehistory_session_parity.py). +from scraper_kit.cian_state_parser import extract_state from sqlalchemy import text from sqlalchemy.orm import Session from app.core.config import settings -from app.services.scrapers.cian_state_parser import extract_state logger = logging.getLogger(__name__) diff --git a/tradein-mvp/backend/app/services/yandex_price_history.py b/tradein-mvp/backend/app/services/yandex_price_history.py index 7665aad2..5b5a7ab1 100644 --- a/tradein-mvp/backend/app/services/yandex_price_history.py +++ b/tradein-mvp/backend/app/services/yandex_price_history.py @@ -26,11 +26,15 @@ from __future__ import annotations import logging from datetime import UTC, datetime, timedelta +# #2306: ScrapedLot migrated to scraper_kit (byte-identical field/method surface to +# legacy app.services.scrapers.base, см. tests/test_scraper_kit_pricehistory_session_parity.py). +# record_yandex_price_history reads lot.* via duck-typing only — no isinstance checks — +# так что kit- и legacy-скрапперы (оба конструируют ScrapedLot по (source, source_id, +# price_rub, price_previous_rub, price_trend)) остаются взаимозаменяемы для этой функции. +from scraper_kit.base import ScrapedLot from sqlalchemy import text from sqlalchemy.orm import Session -from app.services.scrapers.base import ScrapedLot - logger = logging.getLogger(__name__) _SOURCE = "yandex" diff --git a/tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py b/tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py new file mode 100644 index 00000000..32039510 --- /dev/null +++ b/tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py @@ -0,0 +1,214 @@ +"""Parity tests for issue #2306 (Group B of scraper_kit migration epic #2277). + +Migrates the INTERNAL legacy-scraper imports of three product-side files to their +`scraper_kit` equivalents (public API of the three files is unchanged — no caller +elsewhere in the codebase needed updating, see vault audit +`Scraper_Kit_Legacy_Dependency_Audit_0703`): + + - app/services/cian_price_history.py → fetch_detail/save_detail_enrichment now from + `scraper_kit.providers.cian.detail` (was `app.services.scrapers.cian_detail`). + - app/services/cian_session.py → extract_state now from + `scraper_kit.cian_state_parser` (was `app.services.scrapers.cian_state_parser`). + - app/services/yandex_price_history.py → ScrapedLot now from `scraper_kit.base` + (was `app.services.scrapers.base`). + +This file proves the DELTA introduced by the migration specifically — general +cian-provider / ScrapedLot golden-parity is already covered by +`tests/test_scraper_kit_cian_golden_parity.py` and +`tests/test_scraper_kit_base_golden_parity.py`. Uses `tests/support/parity.assert_parity` +per the epic's parity-gate convention (issue #2304). +""" + +from __future__ import annotations + +import asyncio +import dataclasses +import json +import os +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +# Старый app.services.scrapers.cian* / app.core.config.settings требует DATABASE_URL +# при импорте (тот же паттерн, что в test_scraper_kit_cian_golden_parity.py). +os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") + +# Kit — то, на что теперь ссылаются мигрированные файлы (#2306). +from scraper_kit.base import ScrapedLot as KitScrapedLot +from scraper_kit.cian_state_parser import extract_state as kit_extract_state +from scraper_kit.providers.cian.detail import fetch_detail as kit_fetch_detail + +from app.core.config import settings +from app.services.scraper_adapters import RealScraperConfig + +# Legacy — эталон сравнения (эти модули НЕ меняются в #2306). +from app.services.scrapers.base import ScrapedLot as LegacyScrapedLot +from app.services.scrapers.cian_detail import fetch_detail as legacy_fetch_detail +from app.services.scrapers.cian_state_parser import extract_state as legacy_extract_state +from tests.support.parity import assert_parity + +# ── extract_state parity (cian_session.py dependency) ───────────────────────── + + +def _auth_state_html() -> str: + state = {"user": {"isAuthenticated": True, "userId": 42}} + state_json = json.dumps(state, ensure_ascii=False) + return ( + "" + ) + + +def test_extract_state_parity_on_match() -> None: + """cian_session.py._classify_verify_response reads this exact (mfe, key) pair.""" + html = _auth_state_html() + assert_parity( + legacy_fn=legacy_extract_state, + kit_fn=kit_extract_state, + fixtures=[(html, "header-frontend", "initialState")], + ) + + +def test_extract_state_parity_on_miss() -> None: + """Missing mfe/key → both return None identically (not just on the happy path).""" + html = _auth_state_html() + assert_parity( + legacy_fn=legacy_extract_state, + kit_fn=kit_extract_state, + fixtures=[(html, "frontend-offer-card", "defaultState")], + ) + + +# ── ScrapedLot parity (yandex_price_history.py dependency) ──────────────────── + + +def _lot_kwargs() -> dict[str, Any]: + return { + "source": "yandex", + "source_url": "https://realty.yandex.ru/offer/123/", + "source_id": "123", + "price_rub": 5_000_000, + "price_previous_rub": 5_200_000, + "price_trend": "DECREASED", + } + + +def test_scraped_lot_model_dump_parity() -> None: + """record_yandex_price_history duck-types lot.source/.source_id/.price_rub/... + + Legacy- и kit-ScrapedLot должны давать идентичный model_dump() на одном входе — + доказывает, что смена типа импорта в yandex_price_history.py не меняет runtime- + поведение (функция принимает оба класса взаимозаменяемо). + """ + kwargs = _lot_kwargs() + assert_parity( + legacy_fn=lambda kw: LegacyScrapedLot(**kw).model_dump(), + kit_fn=lambda kw: KitScrapedLot(**kw).model_dump(), + fixtures=[kwargs], + ) + + +# ── fetch_detail / DetailEnrichment parity (cian_price_history.py dependency) ── + + +def _detail_html() -> str: + default_state = { + "offerData": { + "offer": { + "cianId": 555111, + "priceChanges": [ + {"changeTime": "2026-05-01T00:00:00Z", "price": 6100000, "diffPercent": -1.2} + ], + }, + "stats": {}, + } + } + ds_json = json.dumps(default_state, ensure_ascii=False) + return ( + "" + ) + + +def _mock_session(html: str) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.text = html + session = MagicMock() + session.get = AsyncMock(return_value=response) + session.close = AsyncMock() + return session + + +def _run_legacy_fetch(url: str, html: str) -> Any: + return dataclasses.asdict(asyncio.run(legacy_fetch_detail(url, session=_mock_session(html)))) + + +def _run_kit_fetch(url: str, html: str) -> Any: + # Точно тот вызов, что теперь делает cian_price_history.backfill_cian_price_history: + # config=RealScraperConfig() добавлен по миграции (#2306); session= общий mock здесь + # только чтобы избежать сетевого I/O — сам config не участвует в session-пути, так + # что этот тест доказывает: инжекция config= не меняет результат парсинга. + result = asyncio.run( + kit_fetch_detail(url, config=RealScraperConfig(), session=_mock_session(html)) + ) + return dataclasses.asdict(result) + + +def test_fetch_detail_with_config_injection_parity() -> None: + html = _detail_html() + url = "https://ekb.cian.ru/sale/flat/555111/" + assert_parity( + legacy_fn=_run_legacy_fetch, + kit_fn=_run_kit_fetch, + fixtures=[(url, html)], + ) + + +def test_fetch_detail_own_session_proxy_wiring_parity(monkeypatch: Any) -> None: + """cian_price_history calls fetch_detail WITHOUT session= (own-session curl_cffi path). + + Kit fetch_detail without config= silently drops the proxy (direct connection) — a + real behaviour regression risk (#806: datacenter-IP gets banned by Cian without a + proxy). This proves config=RealScraperConfig() (our actual call site) constructs + AsyncSession(proxies=...) identically to the legacy direct-settings path. + """ + monkeypatch.setattr(settings, "cian_proxy_url_env", "http://test-proxy.local:8080") + + captured: dict[str, dict[str, Any]] = {} + + def _make_fake_session(name: str) -> Any: + class _FakeSession: + def __init__(self, **kwargs: Any) -> None: + captured[name] = kwargs + + async def get(self, *_args: Any, **_kwargs: Any) -> Any: + response = MagicMock() + response.status_code = 200 + response.text = _detail_html() + return response + + async def close(self) -> None: + pass + + return _FakeSession + + url = "https://ekb.cian.ru/sale/flat/1/" + + with patch("app.services.scrapers.cian_detail.AsyncSession", _make_fake_session("legacy")): + asyncio.run(legacy_fetch_detail(url)) + + with patch("scraper_kit.providers.cian.detail.AsyncSession", _make_fake_session("kit")): + asyncio.run(kit_fetch_detail(url, config=RealScraperConfig())) + + assert captured["legacy"].get("proxies") == captured["kit"].get("proxies") + assert captured["kit"]["proxies"] == { + "http": "http://test-proxy.local:8080", + "https": "http://test-proxy.local:8080", + } -- 2.45.3 From 88b93435fc7135d533a53ccf7f081eea893752e2 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 00:27:08 +0300 Subject: [PATCH 2/2] test(tradein): assert config kwarg on cian price-history fetch_detail call (#2306) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit code-reviewer flagged that neither the new parity test nor test_backfill_wave2.py's 3 cian_price_history tests would catch a regression if config=RealScraperConfig() were accidentally dropped from the fetch_detail call in cian_price_history.py (the proxy-footgun fix from the #2306 migration) — mock_fetch.assert_called_once() doesn't check kwargs. Strengthens all 3 tests (saves_changes / skips_no_changes / handles_fetch_none) to assert isinstance(call_kwargs["config"], RealScraperConfig). Manually verified the guard actually catches the regression: temporarily removed config= from cian_price_history.py:110, confirmed all 3 tests fail, restored the fix, confirmed they pass again. --- .../backend/tests/test_backfill_wave2.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/tradein-mvp/backend/tests/test_backfill_wave2.py b/tradein-mvp/backend/tests/test_backfill_wave2.py index 63c12f1c..bfe72803 100644 --- a/tradein-mvp/backend/tests/test_backfill_wave2.py +++ b/tradein-mvp/backend/tests/test_backfill_wave2.py @@ -235,6 +235,7 @@ async def test_backfill_yandex_addresses_error_on_non_200(): async def test_cian_price_history_saves_changes(): """Happy path: fetch returns DetailEnrichment with price_changes → saved.""" from app.services.cian_price_history import backfill_cian_price_history + from app.services.scraper_adapters import RealScraperConfig from app.services.scrapers.cian_detail import DetailEnrichment enrichment = DetailEnrichment( @@ -277,6 +278,12 @@ async def test_cian_price_history_saves_changes(): assert result.saved == 2 # 2 price_changes assert result.errors == 0 mock_fetch.assert_called_once() + # #2306 regression guard: kit fetch_detail silently drops the Cian proxy + # (direct connection, #806 ban risk) unless config=RealScraperConfig() is + # passed at the call site — assert_called_once() alone wouldn't catch someone + # dropping that kwarg later. + _, call_kwargs = mock_fetch.call_args + assert isinstance(call_kwargs.get("config"), RealScraperConfig) mock_save.assert_called_once() @@ -284,6 +291,7 @@ async def test_cian_price_history_saves_changes(): async def test_cian_price_history_skips_no_changes(): """Fetch succeeds but no price_changes → skipped (not error).""" from app.services.cian_price_history import backfill_cian_price_history + from app.services.scraper_adapters import RealScraperConfig from app.services.scrapers.cian_detail import DetailEnrichment enrichment = DetailEnrichment(cian_id=111, price_changes=[]) @@ -296,7 +304,7 @@ async def test_cian_price_history_skips_no_changes(): "app.services.cian_price_history.fetch_detail", new_callable=AsyncMock, return_value=enrichment, - ), + ) as mock_fetch, patch("app.services.cian_price_history.save_detail_enrichment") as mock_save, patch("app.services.cian_price_history.get_scraper_delay", return_value=0.0), ): @@ -309,12 +317,16 @@ async def test_cian_price_history_skips_no_changes(): assert result.skipped == 1 assert result.saved == 0 mock_save.assert_not_called() + # #2306 regression guard — see test_cian_price_history_saves_changes. + _, call_kwargs = mock_fetch.call_args + assert isinstance(call_kwargs.get("config"), RealScraperConfig) @pytest.mark.asyncio async def test_cian_price_history_handles_fetch_none(): """fetch_detail returns None → errors+1, no crash.""" from app.services.cian_price_history import backfill_cian_price_history + from app.services.scraper_adapters import RealScraperConfig mock_db = MagicMock() rows = [{"id": 30, "source_url": "https://ekb.cian.ru/sale/flat/999/"}] @@ -324,7 +336,7 @@ async def test_cian_price_history_handles_fetch_none(): "app.services.cian_price_history.fetch_detail", new_callable=AsyncMock, return_value=None, - ), + ) as mock_fetch, patch("app.services.cian_price_history.get_scraper_delay", return_value=0.0), ): mock_mappings = MagicMock() @@ -335,6 +347,9 @@ async def test_cian_price_history_handles_fetch_none(): assert result.errors == 1 assert result.saved == 0 + # #2306 regression guard — see test_cian_price_history_saves_changes. + _, call_kwargs = mock_fetch.call_args + assert isinstance(call_kwargs.get("config"), RealScraperConfig) # ============================================================================ -- 2.45.3