gendesign/tradein-mvp/backend/tests/test_scraper_kit_cian_golden_parity.py

"""Golden-parity: `scraper_kit.providers.cian.*` парсинг ≡ ещё живым `app.services.scrapers.cian*`.

Strangler-инвариант (#2133 cian): новая scraper_kit-копия cian-провайдера должна давать
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. Развязка
(settings→ScraperConfig, get_scraper_delay→delay_provider, cian_session→локальная копия)
НЕ меняет распарсенные данные.

`app.services.scrapers.cian` / `cian_detail` удалены (#2397 Part E2, 0 runtime-
импортёров — admin.py/cian_history_backfill.py уже на kit). Parity-тесты SERP
`_parse_serp_html`/`_offer_to_lot`/`_extract_total_offers`/`_format_address` и
detail `fetch_detail` удалены вместе с ними (легаси-эталон исчез).

Оставшееся здесь сравнивает kit ≡ ЕЩЁ ЖИВЫЕ легаси-модули:
  - `extract_state` — `app.services.scrapers.cian_state_parser` (живой, общий парсер state);
  - valuation `_parse_valuation_state` / `estimate_via_cian_valuation` —
    `app.services.scrapers.cian_valuation` (живой);
  - newbuilding pure-экстракторы — `app.services.scrapers.cian_newbuilding` (живой);
  - kit mandatory-config guards (`estimate_via_cian_valuation`, `cian.session`) — kit-only;
  - strangler-guard: kit-провайдер не импортирует `app.*`.

Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
"""

from __future__ import annotations

import asyncio
import dataclasses
import json
import os
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock

import pytest

# Старый app.services.scrapers.cian_state_parser/cian_valuation/cian_newbuilding
# импортируют app.core.config.settings=Settings(), которому нужен DATABASE_URL.
# Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")

# НОВЫЙ (scraper_kit) cian-провайдер.
from scraper_kit.cian_state_parser import extract_state as new_extract_state
from scraper_kit.providers.cian.newbuilding import _extract_chart as new_extract_chart
from scraper_kit.providers.cian.newbuilding import (
    _extract_nested_offers as new_extract_nested_offers,
)
from scraper_kit.providers.cian.newbuilding import (
    _extract_reliability_checks as new_extract_reliability,
)
from scraper_kit.providers.cian.newbuilding import (
    _extract_zhk_url_from_serp as new_extract_zhk_url,
)
from scraper_kit.providers.cian.valuation import (
    _parse_valuation_state as new_parse_valuation,
)
from scraper_kit.providers.cian.valuation import (
    estimate_via_cian_valuation as new_estimate_via_cian_valuation,
)

# #2335 (Group E2): estimate_via_cian_valuation full-pipeline parity + mandatory-config
# regression guard. RealScraperConfig — тот же read-only адаптер над settings, что и
# #2306's cian_price_history.py (см. app/services/scraper_adapters.py).
from app.services.scraper_adapters import RealScraperConfig

# СТАРЫЙ (app.services.scrapers) — ещё живые cian-модули (cian.py/cian_detail.py удалены).
from app.services.scrapers.cian_newbuilding import _extract_chart as old_extract_chart
from app.services.scrapers.cian_newbuilding import (
    _extract_nested_offers as old_extract_nested_offers,
)
from app.services.scrapers.cian_newbuilding import (
    _extract_reliability_checks as old_extract_reliability,
)
from app.services.scrapers.cian_newbuilding import (
    _extract_zhk_url_from_serp as old_extract_zhk_url,
)
from app.services.scrapers.cian_state_parser import extract_state as old_extract_state
from app.services.scrapers.cian_valuation import (
    _parse_valuation_state as old_parse_valuation,
)
from app.services.scrapers.cian_valuation import (
    estimate_via_cian_valuation as old_estimate_via_cian_valuation,
)
from tests.support.parity import assert_parity

# ── extract_state parity (state-парсер, живой в cian_state_parser.py) ────────


def _serp_html_for_state(state: dict[str, Any]) -> str:
    """Обернуть произвольный state в валидный _cianConfig['frontend-serp'].push() HTML."""
    state_json = json.dumps(state, ensure_ascii=False)
    return (
        "<html><body><script>\n"
        "window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];\n"
        "window._cianConfig['frontend-serp'].push({"
        f"key: 'initialState', value: {state_json}, priority: 10, filter: null"
        "});\n</script></body></html>"
    )


def test_extract_state_parity() -> None:
    """app.extract_state ≡ scraper_kit.extract_state на SERP-фикстуре."""
    html = _serp_html_for_state({"results": {"totalOffers": 1, "offers": []}})
    old_state = old_extract_state(html, mfe="frontend-serp", key="initialState")
    new_state = new_extract_state(html, mfe="frontend-serp", key="initialState")
    assert old_state == new_state
    assert new_state is not None


# ── valuation _parse_valuation_state parity (живой cian_valuation.py) ────────


def _valuation_state() -> dict[str, Any]:
    return {
        "user": {"isAuthenticated": True, "userId": 42},
        "estimation": {
            "sale": {
                "data": {
                    "price": 6800000,
                    "accuracy": 87,
                    "priceFrom": 6400000,
                    "priceTo": 7200000,
                    "priceSqm": 130000,
                    "filtersHash": "hash-abc",
                }
            },
            "rent": {
                "data": {
                    "price": 42000,
                    "accuracy": 80,
                    "priceFrom": 39000,
                    "priceTo": 45000,
                    "taxPrice": 5460,
                }
            },
        },
        "estimationChart": {
            "data": {
                "title": {"change": "increase", "changeValue": "8,3%"},
                "chartData": {
                    "data": [
                        {"date": 1714521600000, "price": 6700000, "priceFormatted": "6,7 млн"},
                        {"date": 1717200000000, "price": 6800000, "priceFormatted": "6,8 млн"},
                    ]
                },
            }
        },
        "houseInfo": {
            "data": {
                "items": [
                    {"title": "Год постройки", "value": "2010"},
                    {"title": "Тип дома", "value": "монолит"},
                ]
            }
        },
        "filters": {"houseId": 998877},
        "managementCompany": {"data": {"name": "УК Ромашка", "phones": ["+73430000000"]}},
    }


def test_valuation_parse_state_parity() -> None:
    """_parse_valuation_state → CianValuationResult идентичен (все поля)."""
    state = _valuation_state()
    old_res = old_parse_valuation(state)
    new_res = new_parse_valuation(state)
    assert dataclasses.asdict(old_res) == dataclasses.asdict(new_res)


# ── estimate_via_cian_valuation full-pipeline parity (#2335, Group E2) ───────
#
# test_valuation_parse_state_parity above only covers the pure parsing helper.
# These tests exercise the FULL async call path (cache lookup → load_session →
# curl_cffi fetch → extract_state → auth-check → parse → sanity-check → cache
# write), with config=RealScraperConfig() wired per the #2306 convention
# (app/services/cian_price_history.py), proving the kit path is safe to switch
# to whenever #2337 wires estimator.py's call site.


def _valuation_mfe_html(state: dict[str, Any]) -> str:
    """Wrap a valuation state dict as a valid _cianConfig push() HTML page."""
    state_json = json.dumps(state, ensure_ascii=False)
    return (
        "<html><body><script>\n"
        "window._cianConfig['valuation-for-agent-frontend'] = "
        "window._cianConfig['valuation-for-agent-frontend'] || [];\n"
        "window._cianConfig['valuation-for-agent-frontend'].push({"
        f"key: 'initialState', value: {state_json}, priority: 10, filter: null"
        "});\n</script></body></html>"
    )


def _make_fake_async_session_cls(html: str) -> type:
    """Stand-in for curl_cffi.requests.AsyncSession — returns canned HTML on .get()."""

    class _FakeAsyncSession:
        def __init__(self, *args: Any, **kwargs: Any) -> None:
            pass

        async def __aenter__(self) -> _FakeAsyncSession:
            return self

        async def __aexit__(self, *exc: object) -> bool:
            return False

        async def get(self, url: str, allow_redirects: bool = True) -> SimpleNamespace:
            return SimpleNamespace(status_code=200, text=html)

    return _FakeAsyncSession


def test_estimate_via_cian_valuation_full_pipeline_parity(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Full estimate_via_cian_valuation() pipeline parity — not just the pure parser.

    Mocks curl_cffi.AsyncSession (canned MFE HTML) + load_session (fixed cookies) on
    BOTH legacy and kit modules, cache-miss on a MagicMock db, and drives both async
    call paths with identical inputs (kit wired with config=RealScraperConfig()).
    """
    html = _valuation_mfe_html(_valuation_state())
    fake_session_cls = _make_fake_async_session_cls(html)

    def _run_legacy() -> Any:
        monkeypatch.setattr("app.services.scrapers.cian_valuation.AsyncSession", fake_session_cls)
        monkeypatch.setattr(
            "app.services.scrapers.cian_valuation.load_session",
            lambda db: {"DMIR_AUTH": "token"},
        )
        db = MagicMock()
        db.execute.return_value.mappings.return_value.first.return_value = None  # cache miss
        return asyncio.run(
            old_estimate_via_cian_valuation(
                db,
                address="Свердловская обл, Екатеринбург",
                total_area=50.0,
                rooms_count=2,
                floor=5,
                total_floors=16,
                use_cache=False,
            )
        )

    def _run_kit() -> Any:
        monkeypatch.setattr("scraper_kit.providers.cian.valuation.AsyncSession", fake_session_cls)
        monkeypatch.setattr(
            "scraper_kit.providers.cian.valuation.load_session",
            lambda db, *, config: {"DMIR_AUTH": "token"},
        )
        db = MagicMock()
        db.execute.return_value.mappings.return_value.first.return_value = None  # cache miss
        return asyncio.run(
            new_estimate_via_cian_valuation(
                db,
                config=RealScraperConfig(),
                address="Свердловская обл, Екатеринбург",
                total_area=50.0,
                rooms_count=2,
                floor=5,
                total_floors=16,
                use_cache=False,
            )
        )

    assert_parity(
        legacy_fn=_run_legacy,
        kit_fn=_run_kit,
        fixtures=[()],
    )


def test_kit_estimate_via_cian_valuation_requires_config() -> None:
    """Kit's estimate_via_cian_valuation MUST require config= (no default) — mandatory.

    Regression guard (#2335): config: ScraperConfig has NO default in the kit
    signature, so calling it without config= raises TypeError immediately (at
    argument-binding time, before any coroutine runs). This is intentional and
    must stay this way: estimator.py's call site (#2337) sits inside a blanket
    `except Exception` used for graceful multi-source degradation — a future
    change that gave config= a default would let a wiring mistake silently
    degrade into "source unavailable" instead of raising loudly at call-time.
    See #2306 for the precedent this guards against (silent-degradation footgun
    from an optional dependency parameter).
    """
    db = MagicMock()
    with pytest.raises(TypeError, match="config"):
        new_estimate_via_cian_valuation(  # not awaited — TypeError raises at call time
            db,
            address="Свердловская обл, Екатеринбург",
            total_area=50.0,
            rooms_count=2,
            floor=5,
        )


def test_kit_cian_session_config_requirement_asymmetry() -> None:
    """scraper_kit.providers.cian.session: load_session requires config=, but
    mark_session_invalid does NOT.

    Confirmed by direct inspection for #2335 — the two helpers estimate_via_cian_valuation
    calls internally are NOT symmetric re: config, unlike the assumption in the original
    task brief ("the kit version of those also requires config mandatorily"). Documented
    here as a regression guard so a future refactor that changes either signature is a
    deliberate, visible decision rather than an accidental drift.
    """
    from scraper_kit.providers.cian.session import load_session, mark_session_invalid

    db = MagicMock()
    with pytest.raises(TypeError, match="config"):
        load_session(db)  # missing mandatory config=

    db2 = MagicMock()
    mark_session_invalid(db2, 42)  # no config parameter at all — must NOT raise
    db2.execute.assert_called_once()


# ── newbuilding pure-экстракторы parity (живой cian_newbuilding.py) ──────────


def test_newbuilding_extractors_parity() -> None:
    rv_state = {
        "data": {
            "priceDynamics": {
                "chart": {
                    "data": {"labels": ["2026-01-01", "2026-02-01"], "values": [120000, 125000]}
                }
            }
        }
    }
    assert old_extract_chart(rv_state) == new_extract_chart(rv_state)

    rel_state = {
        "checkStatus": {"status": "reliable", "title": "Надёжный застройщик"},
        "details": [{"title": "Проектная декларация", "type": "doc"}],
    }
    assert old_extract_reliability(rel_state) == new_extract_reliability(rel_state)

    offers_state = {
        "data": {
            "total": {
                "fromDeveloperRooms": [
                    {"roomType": "1", "layouts": [{"id": 1, "area": 38}]},
                    {"roomType": "2", "layouts": [{"id": 2, "area": 55}]},
                ]
            }
        }
    }
    assert old_extract_nested_offers(offers_state) == new_extract_nested_offers(offers_state)

    serp_html = (
        '<h1 data-name="Title">Купить квартиру в '
        '<a href="https://zhk-parkovyy-kvartal-ekb-i.cian.ru/">ЖК «Парковый квартал»</a></h1>'
    )
    assert old_extract_zhk_url(serp_html) == new_extract_zhk_url(serp_html)


# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────


def test_kit_cian_has_no_app_imports() -> None:
    """Ни один модуль scraper_kit.providers.cian.* не импортирует app.* (развязка)."""
    import inspect

    from scraper_kit.providers.cian import detail, newbuilding, serp, session, valuation

    for mod in (serp, detail, newbuilding, valuation, session):
        source = inspect.getsource(mod)
        for line in source.splitlines():
            stripped = line.strip()
            assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
            assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"