"""Golden-parity: `scraper_kit.providers.cian.*` парсинг ≡ ещё живым `app.services.scrapers.cian*`.
Strangler-инвариант (#2133 cian): новая scraper_kit-копия cian-провайдера должна давать
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. Развязка
(settings→ScraperConfig, get_scraper_delay→delay_provider, cian_session→локальная копия)
НЕ меняет распарсенные данные.
`app.services.scrapers.cian` / `cian_detail` удалены (#2397 Part E2, 0 runtime-
импортёров — admin.py/cian_history_backfill.py уже на kit). Parity-тесты SERP
`_parse_serp_html`/`_offer_to_lot`/`_extract_total_offers`/`_format_address` и
detail `fetch_detail` удалены вместе с ними (легаси-эталон исчез).
Оставшееся здесь сравнивает kit ≡ ЕЩЁ ЖИВЫЕ легаси-модули:
- `extract_state` — `app.services.scrapers.cian_state_parser` (живой, общий парсер state);
- valuation `_parse_valuation_state` / `estimate_via_cian_valuation` —
`app.services.scrapers.cian_valuation` (живой);
- newbuilding pure-экстракторы — `app.services.scrapers.cian_newbuilding` (живой);
- kit mandatory-config guards (`estimate_via_cian_valuation`, `cian.session`) — kit-only;
- strangler-guard: kit-провайдер не импортирует `app.*`.
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
"""
from __future__ import annotations
import asyncio
import dataclasses
import json
import os
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import pytest
# Старый app.services.scrapers.cian_state_parser/cian_valuation/cian_newbuilding
# импортируют app.core.config.settings=Settings(), которому нужен DATABASE_URL.
# Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
# НОВЫЙ (scraper_kit) cian-провайдер.
from scraper_kit.cian_state_parser import extract_state as new_extract_state
from scraper_kit.providers.cian.newbuilding import _extract_chart as new_extract_chart
from scraper_kit.providers.cian.newbuilding import (
_extract_nested_offers as new_extract_nested_offers,
)
from scraper_kit.providers.cian.newbuilding import (
_extract_reliability_checks as new_extract_reliability,
)
from scraper_kit.providers.cian.newbuilding import (
_extract_zhk_url_from_serp as new_extract_zhk_url,
)
from scraper_kit.providers.cian.valuation import (
_parse_valuation_state as new_parse_valuation,
)
from scraper_kit.providers.cian.valuation import (
estimate_via_cian_valuation as new_estimate_via_cian_valuation,
)
# #2335 (Group E2): estimate_via_cian_valuation full-pipeline parity + mandatory-config
# regression guard. RealScraperConfig — тот же read-only адаптер над settings, что и
# #2306's cian_price_history.py (см. app/services/scraper_adapters.py).
from app.services.scraper_adapters import RealScraperConfig
# СТАРЫЙ (app.services.scrapers) — ещё живые cian-модули (cian.py/cian_detail.py удалены).
from app.services.scrapers.cian_newbuilding import _extract_chart as old_extract_chart
from app.services.scrapers.cian_newbuilding import (
_extract_nested_offers as old_extract_nested_offers,
)
from app.services.scrapers.cian_newbuilding import (
_extract_reliability_checks as old_extract_reliability,
)
from app.services.scrapers.cian_newbuilding import (
_extract_zhk_url_from_serp as old_extract_zhk_url,
)
from app.services.scrapers.cian_state_parser import extract_state as old_extract_state
from app.services.scrapers.cian_valuation import (
_parse_valuation_state as old_parse_valuation,
)
from app.services.scrapers.cian_valuation import (
estimate_via_cian_valuation as old_estimate_via_cian_valuation,
)
from tests.support.parity import assert_parity
# ── extract_state parity (state-парсер, живой в cian_state_parser.py) ────────
def _serp_html_for_state(state: dict[str, Any]) -> str:
"""Обернуть произвольный state в валидный _cianConfig['frontend-serp'].push() HTML."""
state_json = json.dumps(state, ensure_ascii=False)
return (
""
)
def test_extract_state_parity() -> None:
"""app.extract_state ≡ scraper_kit.extract_state на SERP-фикстуре."""
html = _serp_html_for_state({"results": {"totalOffers": 1, "offers": []}})
old_state = old_extract_state(html, mfe="frontend-serp", key="initialState")
new_state = new_extract_state(html, mfe="frontend-serp", key="initialState")
assert old_state == new_state
assert new_state is not None
# ── valuation _parse_valuation_state parity (живой cian_valuation.py) ────────
def _valuation_state() -> dict[str, Any]:
return {
"user": {"isAuthenticated": True, "userId": 42},
"estimation": {
"sale": {
"data": {
"price": 6800000,
"accuracy": 87,
"priceFrom": 6400000,
"priceTo": 7200000,
"priceSqm": 130000,
"filtersHash": "hash-abc",
}
},
"rent": {
"data": {
"price": 42000,
"accuracy": 80,
"priceFrom": 39000,
"priceTo": 45000,
"taxPrice": 5460,
}
},
},
"estimationChart": {
"data": {
"title": {"change": "increase", "changeValue": "8,3%"},
"chartData": {
"data": [
{"date": 1714521600000, "price": 6700000, "priceFormatted": "6,7 млн"},
{"date": 1717200000000, "price": 6800000, "priceFormatted": "6,8 млн"},
]
},
}
},
"houseInfo": {
"data": {
"items": [
{"title": "Год постройки", "value": "2010"},
{"title": "Тип дома", "value": "монолит"},
]
}
},
"filters": {"houseId": 998877},
"managementCompany": {"data": {"name": "УК Ромашка", "phones": ["+73430000000"]}},
}
def test_valuation_parse_state_parity() -> None:
"""_parse_valuation_state → CianValuationResult идентичен (все поля)."""
state = _valuation_state()
old_res = old_parse_valuation(state)
new_res = new_parse_valuation(state)
assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)
# ── estimate_via_cian_valuation full-pipeline parity (#2335, Group E2) ───────
#
# test_valuation_parse_state_parity above only covers the pure parsing helper.
# These tests exercise the FULL async call path (cache lookup → load_session →
# curl_cffi fetch → extract_state → auth-check → parse → sanity-check → cache
# write), with config=RealScraperConfig() wired per the #2306 convention
# (app/services/cian_price_history.py), proving the kit path is safe to switch
# to whenever #2337 wires estimator.py's call site.
def _valuation_mfe_html(state: dict[str, Any]) -> str:
"""Wrap a valuation state dict as a valid _cianConfig push() HTML page."""
state_json = json.dumps(state, ensure_ascii=False)
return (
""
)
def _make_fake_async_session_cls(html: str) -> type:
"""Stand-in for curl_cffi.requests.AsyncSession — returns canned HTML on .get()."""
class _FakeAsyncSession:
def __init__(self, *args: Any, **kwargs: Any) -> None:
pass
async def __aenter__(self) -> _FakeAsyncSession:
return self
async def __aexit__(self, *exc: object) -> bool:
return False
async def get(self, url: str, allow_redirects: bool = True) -> SimpleNamespace:
return SimpleNamespace(status_code=200, text=html)
return _FakeAsyncSession
def test_estimate_via_cian_valuation_full_pipeline_parity(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""Full estimate_via_cian_valuation() pipeline parity — not just the pure parser.
Mocks curl_cffi.AsyncSession (canned MFE HTML) + load_session (fixed cookies) on
BOTH legacy and kit modules, cache-miss on a MagicMock db, and drives both async
call paths with identical inputs (kit wired with config=RealScraperConfig()).
"""
html = _valuation_mfe_html(_valuation_state())
fake_session_cls = _make_fake_async_session_cls(html)
def _run_legacy() -> Any:
monkeypatch.setattr("app.services.scrapers.cian_valuation.AsyncSession", fake_session_cls)
monkeypatch.setattr(
"app.services.scrapers.cian_valuation.load_session",
lambda db: {"DMIR_AUTH": "token"},
)
db = MagicMock()
db.execute.return_value.mappings.return_value.first.return_value = None # cache miss
return asyncio.run(
old_estimate_via_cian_valuation(
db,
address="Свердловская обл, Екатеринбург",
total_area=50.0,
rooms_count=2,
floor=5,
total_floors=16,
use_cache=False,
)
)
def _run_kit() -> Any:
monkeypatch.setattr("scraper_kit.providers.cian.valuation.AsyncSession", fake_session_cls)
monkeypatch.setattr(
"scraper_kit.providers.cian.valuation.load_session",
lambda db, *, config: {"DMIR_AUTH": "token"},
)
db = MagicMock()
db.execute.return_value.mappings.return_value.first.return_value = None # cache miss
return asyncio.run(
new_estimate_via_cian_valuation(
db,
config=RealScraperConfig(),
address="Свердловская обл, Екатеринбург",
total_area=50.0,
rooms_count=2,
floor=5,
total_floors=16,
use_cache=False,
)
)
assert_parity(
legacy_fn=_run_legacy,
kit_fn=_run_kit,
fixtures=[()],
)
def test_kit_estimate_via_cian_valuation_requires_config() -> None:
"""Kit's estimate_via_cian_valuation MUST require config= (no default) — mandatory.
Regression guard (#2335): config: ScraperConfig has NO default in the kit
signature, so calling it without config= raises TypeError immediately (at
argument-binding time, before any coroutine runs). This is intentional and
must stay this way: estimator.py's call site (#2337) sits inside a blanket
`except Exception` used for graceful multi-source degradation — a future
change that gave config= a default would let a wiring mistake silently
degrade into "source unavailable" instead of raising loudly at call-time.
See #2306 for the precedent this guards against (silent-degradation footgun
from an optional dependency parameter).
"""
db = MagicMock()
with pytest.raises(TypeError, match="config"):
new_estimate_via_cian_valuation( # not awaited — TypeError raises at call time
db,
address="Свердловская обл, Екатеринбург",
total_area=50.0,
rooms_count=2,
floor=5,
)
def test_kit_cian_session_config_requirement_asymmetry() -> None:
"""scraper_kit.providers.cian.session: load_session requires config=, but
mark_session_invalid does NOT.
Confirmed by direct inspection for #2335 — the two helpers estimate_via_cian_valuation
calls internally are NOT symmetric re: config, unlike the assumption in the original
task brief ("the kit version of those also requires config mandatorily"). Documented
here as a regression guard so a future refactor that changes either signature is a
deliberate, visible decision rather than an accidental drift.
"""
from scraper_kit.providers.cian.session import load_session, mark_session_invalid
db = MagicMock()
with pytest.raises(TypeError, match="config"):
load_session(db) # missing mandatory config=
db2 = MagicMock()
mark_session_invalid(db2, 42) # no config parameter at all — must NOT raise
db2.execute.assert_called_once()
# ── newbuilding pure-экстракторы parity (живой cian_newbuilding.py) ──────────
def test_newbuilding_extractors_parity() -> None:
rv_state = {
"data": {
"priceDynamics": {
"chart": {
"data": {"labels": ["2026-01-01", "2026-02-01"], "values": [120000, 125000]}
}
}
}
}
assert old_extract_chart(rv_state) == new_extract_chart(rv_state)
rel_state = {
"checkStatus": {"status": "reliable", "title": "Надёжный застройщик"},
"details": [{"title": "Проектная декларация", "type": "doc"}],
}
assert old_extract_reliability(rel_state) == new_extract_reliability(rel_state)
offers_state = {
"data": {
"total": {
"fromDeveloperRooms": [
{"roomType": "1", "layouts": [{"id": 1, "area": 38}]},
{"roomType": "2", "layouts": [{"id": 2, "area": 55}]},
]
}
}
}
assert old_extract_nested_offers(offers_state) == new_extract_nested_offers(offers_state)
serp_html = (
''
)
assert old_extract_zhk_url(serp_html) == new_extract_zhk_url(serp_html)
# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────
def test_kit_cian_has_no_app_imports() -> None:
"""Ни один модуль scraper_kit.providers.cian.* не импортирует app.* (развязка)."""
import inspect
from scraper_kit.providers.cian import detail, newbuilding, serp, session, valuation
for mod in (serp, detail, newbuilding, valuation, session):
source = inspect.getsource(mod)
for line in source.splitlines():
stripped = line.strip()
assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"