"""Golden-parity: `scraper_kit.providers.yandex.*` парсинг ≡ `app.services.scrapers.yandex_*`. Strangler-инвариант (#2133): новая scraper_kit-копия yandex-провайдера должна давать БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе. Развязка (settings→ScraperConfig, get_scraper_delay→delay_provider, house_type_normalizer→providers.yandex.shared, base/browser_fetcher/helpers→scraper_kit.*) НЕ меняет распарсенные данные. Гоняем ОБА модуля на одних фикстурах и сравниваем результат: - SERP `_entity_to_lot` (gate-API entity → ScrapedLot) на репрезентативных entity; - SERP `_parse_gate_json` (payload → list[ScrapedLot]) с новостройкой/вторичкой; - `normalize_house_type` (SCREAMING/camelCase → канон); - detail `parse` (INITIAL_STATE.offerCard.card + JSON-LD → DetailEnrichment) на реальных offer-HTML фикстурах; - valuation `parse` (house-meta + history → YandexValuationResult) на синтетике; - newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) на синтетике; - `_build_url` (gate-API URL builder) — детерминированность. Идентичность результата = kit-парсер верно скопирован (развязка не задела логику). """ from __future__ import annotations import inspect import os from pathlib import Path from types import SimpleNamespace from typing import Any # Старый app.services.scrapers.yandex_* импортирует app.core.config.settings=Settings(), # которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен. os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db") # НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) yandex-парсеры — оба в одном # тесте; ruff сортирует scraper_kit (first-party) выше app-импортов. from scraper_kit.providers.yandex.detail import YandexDetailScraper as NewDetailScraper from scraper_kit.providers.yandex.newbuilding import ( YandexNewbuildingScraper as NewNewbuildingScraper, ) from scraper_kit.providers.yandex.serp import YandexRealtyScraper as NewRealtyScraper from scraper_kit.providers.yandex.serp import _entity_to_lot as new_entity_to_lot from scraper_kit.providers.yandex.serp import _parse_gate_json as new_parse_gate_json from scraper_kit.providers.yandex.shared import normalize_house_type as new_normalize_house_type from scraper_kit.providers.yandex.valuation import ( YandexValuationScraper as NewValuationScraper, ) from app.services.scrapers.house_type_normalizer import ( normalize_house_type as old_normalize_house_type, ) from app.services.scrapers.yandex_detail import YandexDetailScraper as OldDetailScraper from app.services.scrapers.yandex_newbuilding import ( YandexNewbuildingScraper as OldNewbuildingScraper, ) from app.services.scrapers.yandex_realty import YandexRealtyScraper as OldRealtyScraper from app.services.scrapers.yandex_realty import _entity_to_lot as old_entity_to_lot from app.services.scrapers.yandex_realty import _parse_gate_json as old_parse_gate_json from app.services.scrapers.yandex_valuation import ( YandexValuationScraper as OldValuationScraper, ) _FIXTURES = Path(__file__).parent / "fixtures" # Инжектируемый конфиг для kit-скрапперов (duck-typed namespace, structurally # удовлетворяет scraper_kit.contracts.ScraperConfig в объёме читаемых полей). _KIT_CONFIG = SimpleNamespace( scraper_proxy_url=None, yandex_proxy_rotate_url=None, avito_proxy_rotate_url=None, ) # ── Fixtures: gate-API entities (mirror test_yandex_realty_serp) ───────────── _ENTITY_FULL: dict = { "offerId": 4740475460451078271, "url": "//realty.yandex.ru/offer/4740475460451078271", "price": {"currency": "RUR", "value": 4500000, "valuePerPart": 119048}, "area": {"value": 37.8, "unit": "SQUARE_METER"}, "livingSpace": {"value": 20, "unit": "SQUARE_METER"}, "kitchenSpace": {"value": 12.7, "unit": "SQUARE_METER"}, "roomsTotal": 1, "floorsOffered": [8], "floorsTotal": 25, "ceilingHeight": 2.7, "building": { "builtYear": 2016, "buildingType": "MONOLIT", "siteId": 280938, "siteName": "Peremen", }, "location": { "geocoderAddress": "Ekaterinburg, ulitsa Evgenia Savkova, 8", "point": {"latitude": 56.79512, "longitude": 60.49186}, }, "mainImages": [ "//avatars.mds.yandex.net/get-realty-offers/11904018/abc/main", "//avatars.mds.yandex.net/get-realty-offers/14717586/def/main", ], } _ENTITY_STUDIO: dict = { "offerId": 9999999, "url": "//realty.yandex.ru/offer/9999999", "price": {"value": 3200000, "valuePerPart": 114285}, "area": {"value": 28.0}, "livingSpace": None, "kitchenSpace": None, "roomsTotal": None, # studio -> rooms=0 "floorsOffered": [2], "floorsTotal": 9, "ceilingHeight": None, "building": {"builtYear": 2010, "buildingType": "PANEL", "siteId": None, "siteName": None}, "location": { "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 5", "point": {"latitude": 56.838, "longitude": 60.597}, }, "mainImages": [], } _ENTITY_RICH_OWNER: dict = { "offerId": 5500000000000000001, "url": "//realty.yandex.ru/offer/5500000000000000001", "creationDate": "2026-05-24T15:10:27Z", "description": "Продаётся светлая квартира с видом на парк.", "author": {"category": "OWNER", "agentName": "Иван", "organization": None}, "price": {"value": 7200000, "valuePerPart": 150000, "trend": "DECREASED", "previous": 7500000}, "predictions": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}}, "area": {"value": 48.0}, "roomsTotal": 2, "floorsOffered": [5], "floorsTotal": 12, "building": {"builtYear": 2018, "buildingType": "MONOLIT"}, "location": { "geocoderAddress": "Ekaterinburg, ulitsa Mira, 10", "point": {"latitude": 56.84, "longitude": 60.61}, "metroList": [ {"name": "Ploshchad 1905 goda", "timeToMetro": 7}, {"name": "Geologicheskaya", "timeToMetro": 12}, ], }, "mainImages": [], } _ENTITY_RICH_AGENCY: dict = { "offerId": 5500000000000000002, "url": "//realty.yandex.ru/offer/5500000000000000002", "creationDate": "2026-06-01T09:00:00Z", "description": "Агентская продажа.", "author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"}, "price": {"value": 9900000, "trend": "UNCHANGED"}, "area": {"value": 60.0}, "roomsTotal": 3, "floorsOffered": [3], "floorsTotal": 9, "building": {"builtYear": 2012, "buildingType": "BRICK"}, "location": { "geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1", "point": {"latitude": 56.83, "longitude": 60.6}, "metro": {"name": "Dinamo", "timeToMetro": 4}, }, "mainImages": [], } _ALL_ENTITIES = [_ENTITY_FULL, _ENTITY_STUDIO, _ENTITY_RICH_OWNER, _ENTITY_RICH_AGENCY] def _make_gate_payload(entities: list, pager: dict | None = None) -> dict: if pager is None: pager = {"page": 0, "pageSize": 20, "totalItems": len(entities), "totalPages": 1} return {"response": {"search": {"offers": {"entities": entities, "pager": pager}}}} def _dump_lots(lots: list[Any]) -> list[dict[str, Any]]: return [lot.model_dump() for lot in lots] # ── SERP _entity_to_lot parity ─────────────────────────────────────────────── def test_entity_to_lot_parity_all_entities() -> None: """gate-API entity → ScrapedLot идентичен на всех репрезентативных entity.""" for entity in _ALL_ENTITIES: old_lot = old_entity_to_lot(entity) new_lot = new_entity_to_lot(entity) assert (old_lot is None) == (new_lot is None) assert old_lot is not None and new_lot is not None assert old_lot.model_dump() == new_lot.model_dump() def test_entity_to_lot_parity_new_flat_segments() -> None: """new_flat=YES/NO → listing_segment идентичен (новостройка/вторичка).""" for new_flat in ("NO", "YES"): old_lot = old_entity_to_lot(_ENTITY_FULL, new_flat=new_flat) new_lot = new_entity_to_lot(_ENTITY_FULL, new_flat=new_flat) assert old_lot is not None and new_lot is not None assert old_lot.model_dump() == new_lot.model_dump() def test_entity_to_lot_parity_ceiling_out_of_range() -> None: """ceilingHeight мусор (18 / 1.6) → одинаково дропается в None (#2007).""" for bad_ceiling in (18, 1.6): entity = {**_ENTITY_FULL, "ceilingHeight": bad_ceiling} old_lot = old_entity_to_lot(entity) new_lot = new_entity_to_lot(entity) assert old_lot is not None and new_lot is not None assert old_lot.model_dump() == new_lot.model_dump() # ── SERP _parse_gate_json parity ───────────────────────────────────────────── def test_parse_gate_json_parity() -> None: """payload → list[ScrapedLot] идентичен (вторичка и новостройка).""" payload = _make_gate_payload(_ALL_ENTITIES) for new_flat in ("NO", "YES"): old_lots = old_parse_gate_json(payload, page_param=2, new_flat=new_flat) new_lots = new_parse_gate_json(payload, page_param=2, new_flat=new_flat) assert len(old_lots) == len(new_lots) > 0 assert _dump_lots(old_lots) == _dump_lots(new_lots) # ── normalize_house_type parity ────────────────────────────────────────────── def test_normalize_house_type_parity() -> None: """SCREAMING / camelCase / канон / мусор → идентичный результат.""" raws = [ None, "", "MONOLIT", "BRICK", "PANEL", "MONOLIT_BRICK", "monolithBrick", "gasSilicateBlock", "monolith_brick", "monolith", "stalin", "OTHER", " MONOLIT ", ] for raw in raws: assert old_normalize_house_type(raw) == new_normalize_house_type(raw) # ── detail parse parity (реальные offer-HTML фикстуры) ─────────────────────── _SECONDARY_ID = "3402418396407468801" _STUDIO_ID = "7811691822126445498" def _load_offer(offer_id: str) -> str: return (_FIXTURES / f"yandex_offer_{offer_id}.html").read_text(encoding="utf-8") def _offer_url(offer_id: str) -> str: return f"https://realty.yandex.ru/offer/{offer_id}/" def test_detail_parse_parity() -> None: """INITIAL_STATE.offerCard.card + JSON-LD → DetailEnrichment идентичен.""" old_scraper = OldDetailScraper() new_scraper = NewDetailScraper() for offer_id in (_SECONDARY_ID, _STUDIO_ID): html = _load_offer(offer_id) url = _offer_url(offer_id) old_res = old_scraper.parse(html, offer_url=url) new_res = new_scraper.parse(html, offer_url=url) assert old_res is not None and new_res is not None assert old_res.model_dump() == new_res.model_dump() # ── valuation parse parity (синтетика) ─────────────────────────────────────── _VALUATION_HTML = """ Дом 2016 года 25 этажей 2,7 м потолки Лифт 42 объекта Панорама 48,5 м², 1-комнатная, 5 этаж 5,1 млн ₽ 105 155 ₽ за м² 23.10.2023 В экспозиции 945 дней Снято 24.05.2024 32 м², студия, 2 этаж 3,2 млн ₽ 100 000 ₽ за м² 01.02.2024 В продаже """ def test_valuation_parse_parity() -> None: """house-meta + history (chunked-text fallback) → идентичный результат.""" old_scraper = OldValuationScraper() new_scraper = NewValuationScraper(config=_KIT_CONFIG) kwargs = dict( address="Екатеринбург, ул. Ленина, 5", offer_category="APARTMENT", offer_type="SELL", page=1, source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address=test", ) old_res = old_scraper.parse(_VALUATION_HTML, **kwargs) new_res = new_scraper.parse(_VALUATION_HTML, **kwargs) assert old_res.model_dump() == new_res.model_dump() # ── newbuilding parse parity (синтетика) ───────────────────────────────────── _NEWBUILDING_HTML = """

ЖК «Татлин»

Екатеринбург, ул. Черепанова 4.3 из 5 1505 оценок Смотреть все 353 отзыва 56.855312 60.576668

О комплексе

Дом комфорт-класса, монолитный. Введён в эксплуатацию в июнь 2023. 35-этажные башни на участке 1,5 га. Три корпуса.
""" def test_newbuilding_parse_parity() -> None: """ЖК-лендинг → YandexNewbuildingInfo идентичен.""" old_scraper = OldNewbuildingScraper() new_scraper = NewNewbuildingScraper() old_res = old_scraper.parse( _NEWBUILDING_HTML, jk_slug="tatlin", jk_id="1592987", source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", ) new_res = new_scraper.parse( _NEWBUILDING_HTML, jk_slug="tatlin", jk_id="1592987", source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/", ) assert old_res.model_dump() == new_res.model_dump() # ── _build_url parity ──────────────────────────────────────────────────────── def test_build_url_parity() -> None: """gate-API URL builder идентичен на репрезентативных комбо.""" old_scraper = OldRealtyScraper() new_scraper = NewRealtyScraper(config=_KIT_CONFIG) combos = [ dict(page=1), dict(page=3, rooms="2", price_min=5_000_000, price_max=7_000_000), dict(page=1, rooms="studio", new_flat="YES"), dict(page=2, rooms="4+", price_min=25_000_000), ] for combo in combos: assert old_scraper._build_url(**combo) == new_scraper._build_url(**combo) # ── strangler-guard: kit-провайдер не импортирует app.* ────────────────────── def test_kit_yandex_has_no_app_imports() -> None: """Ни один модуль scraper_kit.providers.yandex.* не импортирует app.* (развязка).""" from scraper_kit.providers.yandex import detail, newbuilding, serp, shared, valuation for mod in (serp, detail, valuation, newbuilding, shared): source = inspect.getsource(mod) for line in source.splitlines(): stripped = line.strip() assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}" assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"