gendesign/tradein-mvp/backend/tests/test_scraper_kit_yandex_golden_parity.py

"""Golden-parity: `scraper_kit.providers.yandex.*` парсинг ≡ `app.services.scrapers.yandex_*`.

Strangler-инвариант (#2133): новая scraper_kit-копия yandex-провайдера должна давать
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
Развязка (settings→ScraperConfig, get_scraper_delay→delay_provider,
house_type_normalizer→providers.yandex.shared, base/browser_fetcher/helpers→scraper_kit.*)
НЕ меняет распарсенные данные.

Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
  - SERP `_entity_to_lot` (gate-API entity → ScrapedLot) на репрезентативных entity;
  - SERP `_parse_gate_json` (payload → list[ScrapedLot]) с новостройкой/вторичкой;
  - `normalize_house_type` (SCREAMING/camelCase → канон);
  - newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) на синтетике;
  - `_build_url` (gate-API URL builder) — детерминированность.

Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).

Легаси `app.services.scrapers.yandex_detail` / `yandex_valuation` удалены (#2277
финальный шаг scraper_kit-миграции, 0 runtime importers) — detail/valuation
parity-проверки убраны отсюда вместе с ними. Kit-side coverage для detail/valuation
`.parse()` сохранена без изменений в `tests/test_yandex_detail.py` /
`tests/test_yandex_detail_structural.py` / `tests/test_yandex_valuation.py` /
`tests/test_yandex_valuation_save.py` (переведены на kit-эквиваленты тем же PR).
`yandex_realty` / `yandex_newbuilding` / `house_type_normalizer` остаются легаси
(не мигрированы) — их parity-проверки не тронуты.
"""

from __future__ import annotations

import inspect
import os
from types import SimpleNamespace
from typing import Any

# Старый app.services.scrapers.yandex_* импортирует app.core.config.settings=Settings(),
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")

# НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) yandex-парсеры — оба в одном
# тесте; ruff сортирует scraper_kit (first-party) выше app-импортов.
from scraper_kit.providers.yandex.newbuilding import (
    YandexNewbuildingScraper as NewNewbuildingScraper,
)
from scraper_kit.providers.yandex.serp import YandexRealtyScraper as NewRealtyScraper
from scraper_kit.providers.yandex.serp import _entity_to_lot as new_entity_to_lot
from scraper_kit.providers.yandex.serp import _parse_gate_json as new_parse_gate_json
from scraper_kit.providers.yandex.shared import normalize_house_type as new_normalize_house_type

from app.services.scrapers.house_type_normalizer import (
    normalize_house_type as old_normalize_house_type,
)
from app.services.scrapers.yandex_newbuilding import (
    YandexNewbuildingScraper as OldNewbuildingScraper,
)
from app.services.scrapers.yandex_realty import YandexRealtyScraper as OldRealtyScraper
from app.services.scrapers.yandex_realty import _entity_to_lot as old_entity_to_lot
from app.services.scrapers.yandex_realty import _parse_gate_json as old_parse_gate_json

# Инжектируемый конфиг для kit-скрапперов (duck-typed namespace, structurally
# удовлетворяет scraper_kit.contracts.ScraperConfig в объёме читаемых полей).
_KIT_CONFIG = SimpleNamespace(
    scraper_proxy_url=None,
    yandex_proxy_rotate_url=None,
    avito_proxy_rotate_url=None,
)


# ── Fixtures: gate-API entities (mirror test_yandex_realty_serp) ─────────────

_ENTITY_FULL: dict = {
    "offerId": 4740475460451078271,
    "url": "//realty.yandex.ru/offer/4740475460451078271",
    "price": {"currency": "RUR", "value": 4500000, "valuePerPart": 119048},
    "area": {"value": 37.8, "unit": "SQUARE_METER"},
    "livingSpace": {"value": 20, "unit": "SQUARE_METER"},
    "kitchenSpace": {"value": 12.7, "unit": "SQUARE_METER"},
    "roomsTotal": 1,
    "floorsOffered": [8],
    "floorsTotal": 25,
    "ceilingHeight": 2.7,
    "building": {
        "builtYear": 2016,
        "buildingType": "MONOLIT",
        "siteId": 280938,
        "siteName": "Peremen",
    },
    "location": {
        "geocoderAddress": "Ekaterinburg, ulitsa Evgenia Savkova, 8",
        "point": {"latitude": 56.79512, "longitude": 60.49186},
    },
    "mainImages": [
        "//avatars.mds.yandex.net/get-realty-offers/11904018/abc/main",
        "//avatars.mds.yandex.net/get-realty-offers/14717586/def/main",
    ],
}

_ENTITY_STUDIO: dict = {
    "offerId": 9999999,
    "url": "//realty.yandex.ru/offer/9999999",
    "price": {"value": 3200000, "valuePerPart": 114285},
    "area": {"value": 28.0},
    "livingSpace": None,
    "kitchenSpace": None,
    "roomsTotal": None,  # studio -> rooms=0
    "floorsOffered": [2],
    "floorsTotal": 9,
    "ceilingHeight": None,
    "building": {"builtYear": 2010, "buildingType": "PANEL", "siteId": None, "siteName": None},
    "location": {
        "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 5",
        "point": {"latitude": 56.838, "longitude": 60.597},
    },
    "mainImages": [],
}

_ENTITY_RICH_OWNER: dict = {
    "offerId": 5500000000000000001,
    "url": "//realty.yandex.ru/offer/5500000000000000001",
    "creationDate": "2026-05-24T15:10:27Z",
    "description": "Продаётся светлая квартира с видом на парк.",
    "author": {"category": "OWNER", "agentName": "Иван", "organization": None},
    "price": {"value": 7200000, "valuePerPart": 150000, "trend": "DECREASED", "previous": 7500000},
    "predictions": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}},
    "area": {"value": 48.0},
    "roomsTotal": 2,
    "floorsOffered": [5],
    "floorsTotal": 12,
    "building": {"builtYear": 2018, "buildingType": "MONOLIT"},
    "location": {
        "geocoderAddress": "Ekaterinburg, ulitsa Mira, 10",
        "point": {"latitude": 56.84, "longitude": 60.61},
        "metroList": [
            {"name": "Ploshchad 1905 goda", "timeToMetro": 7},
            {"name": "Geologicheskaya", "timeToMetro": 12},
        ],
    },
    "mainImages": [],
}

_ENTITY_RICH_AGENCY: dict = {
    "offerId": 5500000000000000002,
    "url": "//realty.yandex.ru/offer/5500000000000000002",
    "creationDate": "2026-06-01T09:00:00Z",
    "description": "Агентская продажа.",
    "author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"},
    "price": {"value": 9900000, "trend": "UNCHANGED"},
    "area": {"value": 60.0},
    "roomsTotal": 3,
    "floorsOffered": [3],
    "floorsTotal": 9,
    "building": {"builtYear": 2012, "buildingType": "BRICK"},
    "location": {
        "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1",
        "point": {"latitude": 56.83, "longitude": 60.6},
        "metro": {"name": "Dinamo", "timeToMetro": 4},
    },
    "mainImages": [],
}

# #2235: novostroyka offer whose gate-API `url` is a developer/redirect landing
# (shared by many flats). Both parsers must canonicalise it to offer_id-based url.
_ENTITY_PARTNER_URL: dict = {
    "offerId": 7416316697684447350,
    "url": "http://na100.pro/go.php?link=LtWgA6kWd3xjlrUBCLl6VGjo",
    "price": {"value": 7790000, "valuePerPart": 130000},
    "area": {"value": 60.0},
    "roomsTotal": 2,
    "floorsOffered": [5],
    "floorsTotal": 16,
    "building": {"builtYear": 2024, "buildingType": "MONOLIT", "siteId": 999001},
    "location": {
        "geocoderAddress": "Ekaterinburg, ulitsa Titova, 1",
        "point": {"latitude": 56.8, "longitude": 60.6},
    },
    "mainImages": [],
}

_ALL_ENTITIES = [
    _ENTITY_FULL,
    _ENTITY_STUDIO,
    _ENTITY_RICH_OWNER,
    _ENTITY_RICH_AGENCY,
    _ENTITY_PARTNER_URL,
]


def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
    if pager is None:
        pager = {"page": 0, "pageSize": 20, "totalItems": len(entities), "totalPages": 1}
    return {"response": {"search": {"offers": {"entities": entities, "pager": pager}}}}


def _dump_lots(lots: list[Any]) -> list[dict[str, Any]]:
    return [lot.model_dump() for lot in lots]


# ── SERP _entity_to_lot parity ───────────────────────────────────────────────


def test_entity_to_lot_parity_all_entities() -> None:
    """gate-API entity → ScrapedLot идентичен на всех репрезентативных entity."""
    for entity in _ALL_ENTITIES:
        old_lot = old_entity_to_lot(entity)
        new_lot = new_entity_to_lot(entity)
        assert (old_lot is None) == (new_lot is None)
        assert old_lot is not None and new_lot is not None
        assert old_lot.model_dump() == new_lot.model_dump()


def test_entity_to_lot_parity_new_flat_segments() -> None:
    """new_flat=YES/NO → listing_segment идентичен (новостройка/вторичка)."""
    for new_flat in ("NO", "YES"):
        old_lot = old_entity_to_lot(_ENTITY_FULL, new_flat=new_flat)
        new_lot = new_entity_to_lot(_ENTITY_FULL, new_flat=new_flat)
        assert old_lot is not None and new_lot is not None
        assert old_lot.model_dump() == new_lot.model_dump()


def test_entity_to_lot_parity_ceiling_out_of_range() -> None:
    """ceilingHeight мусор (18 / 1.6) → одинаково дропается в None (#2007)."""
    for bad_ceiling in (18, 1.6):
        entity = {**_ENTITY_FULL, "ceilingHeight": bad_ceiling}
        old_lot = old_entity_to_lot(entity)
        new_lot = new_entity_to_lot(entity)
        assert old_lot is not None and new_lot is not None
        assert old_lot.model_dump() == new_lot.model_dump()


# ── SERP _parse_gate_json parity ─────────────────────────────────────────────


def test_parse_gate_json_parity() -> None:
    """payload → list[ScrapedLot] идентичен (вторичка и новостройка)."""
    payload = _make_gate_payload(_ALL_ENTITIES)
    for new_flat in ("NO", "YES"):
        old_lots = old_parse_gate_json(payload, page_param=2, new_flat=new_flat)
        new_lots = new_parse_gate_json(payload, page_param=2, new_flat=new_flat)
        assert len(old_lots) == len(new_lots) > 0
        assert _dump_lots(old_lots) == _dump_lots(new_lots)


# ── normalize_house_type parity ──────────────────────────────────────────────


def test_normalize_house_type_parity() -> None:
    """SCREAMING / camelCase / канон / мусор → идентичный результат."""
    raws = [
        None,
        "",
        "MONOLIT",
        "BRICK",
        "PANEL",
        "MONOLIT_BRICK",
        "monolithBrick",
        "gasSilicateBlock",
        "monolith_brick",
        "monolith",
        "stalin",
        "OTHER",
        "  MONOLIT  ",
    ]
    for raw in raws:
        assert old_normalize_house_type(raw) == new_normalize_house_type(raw)


# ── newbuilding parse parity (синтетика) ─────────────────────────────────────

_NEWBUILDING_HTML = """
<html><body>
<h1>ЖК «Татлин»</h1>
Екатеринбург, ул. Черепанова 4.3 из 5 1505 оценок Смотреть все 353 отзыва
56.855312 60.576668
<h2>О комплексе</h2>
<div>Дом комфорт-класса, монолитный. Введён в эксплуатацию в июнь 2023.
35-этажные башни на участке 1,5 га. Три корпуса.</div>
</body></html>
"""


def test_newbuilding_parse_parity() -> None:
    """ЖК-лендинг → YandexNewbuildingInfo идентичен."""
    old_scraper = OldNewbuildingScraper()
    new_scraper = NewNewbuildingScraper()
    old_res = old_scraper.parse(
        _NEWBUILDING_HTML,
        jk_slug="tatlin",
        jk_id="1592987",
        source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
    )
    new_res = new_scraper.parse(
        _NEWBUILDING_HTML,
        jk_slug="tatlin",
        jk_id="1592987",
        source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
    )
    assert old_res.model_dump() == new_res.model_dump()


# ── _build_url parity ────────────────────────────────────────────────────────


def test_build_url_parity() -> None:
    """gate-API URL builder идентичен на репрезентативных комбо."""
    old_scraper = OldRealtyScraper()
    new_scraper = NewRealtyScraper(config=_KIT_CONFIG)
    combos = [
        dict(page=1),
        dict(page=3, rooms="2", price_min=5_000_000, price_max=7_000_000),
        dict(page=1, rooms="studio", new_flat="YES"),
        dict(page=2, rooms="4+", price_min=25_000_000),
    ]
    for combo in combos:
        assert old_scraper._build_url(**combo) == new_scraper._build_url(**combo)


# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────


def test_kit_yandex_has_no_app_imports() -> None:
    """Ни один модуль scraper_kit.providers.yandex.* не импортирует app.* (развязка)."""
    from scraper_kit.providers.yandex import detail, newbuilding, serp, shared, valuation

    for mod in (serp, detail, valuation, newbuilding, shared):
        source = inspect.getsource(mod)
        for line in source.splitlines():
            stripped = line.strip()
            assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
            assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"