lekss361
3be8062062
Some checks failed
Deploy Trade-In / changes (push) Successful in 11s
Deploy Trade-In / test (push) Failing after 1m31s
Deploy Trade-In / build-frontend (push) Has been skipped
Deploy Trade-In / build-browser (push) Has been skipped
Deploy Trade-In / build-backend (push) Has been skipped
Deploy Trade-In / deploy (push) Has been skipped
364 lines
16 KiB
Python
364 lines
16 KiB
Python
"""Golden-parity: `scraper_kit.providers.yandex.*` парсинг ≡ `app.services.scrapers.yandex_*`.
|
||
|
||
Strangler-инвариант (#2133): новая scraper_kit-копия yandex-провайдера должна давать
|
||
БАЙТ-ИДЕНТИЧНЫЙ результат парсинга старому боевому коду на одинаковом входе.
|
||
Развязка (settings→ScraperConfig, get_scraper_delay→delay_provider,
|
||
house_type_normalizer→providers.yandex.shared, base/browser_fetcher/helpers→scraper_kit.*)
|
||
НЕ меняет распарсенные данные.
|
||
|
||
Гоняем ОБА модуля на одних фикстурах и сравниваем результат:
|
||
- SERP `_entity_to_lot` (gate-API entity → ScrapedLot) на репрезентативных entity;
|
||
- SERP `_parse_gate_json` (payload → list[ScrapedLot]) с новостройкой/вторичкой;
|
||
- `normalize_house_type` (SCREAMING/camelCase → канон);
|
||
- detail `parse` (INITIAL_STATE.offerCard.card + JSON-LD → DetailEnrichment) на
|
||
реальных offer-HTML фикстурах;
|
||
- valuation `parse` (house-meta + history → YandexValuationResult) на синтетике;
|
||
- newbuilding `parse` (ЖК-лендинг → YandexNewbuildingInfo) на синтетике;
|
||
- `_build_url` (gate-API URL builder) — детерминированность.
|
||
|
||
Идентичность результата = kit-парсер верно скопирован (развязка не задела логику).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import inspect
|
||
import os
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
from typing import Any
|
||
|
||
# Старый app.services.scrapers.yandex_* импортирует app.core.config.settings=Settings(),
|
||
# которому нужен DATABASE_URL. Офлайн-парсинг БД не трогает — фиктивный DSN достаточен.
|
||
os.environ.setdefault("DATABASE_URL", "postgresql+psycopg://test:test@localhost/test_db")
|
||
|
||
# НОВЫЙ (scraper_kit) и СТАРЫЙ (app.services.scrapers) yandex-парсеры — оба в одном
|
||
# тесте; ruff сортирует scraper_kit (first-party) выше app-импортов.
|
||
from scraper_kit.providers.yandex.detail import YandexDetailScraper as NewDetailScraper
|
||
from scraper_kit.providers.yandex.newbuilding import (
|
||
YandexNewbuildingScraper as NewNewbuildingScraper,
|
||
)
|
||
from scraper_kit.providers.yandex.serp import YandexRealtyScraper as NewRealtyScraper
|
||
from scraper_kit.providers.yandex.serp import _entity_to_lot as new_entity_to_lot
|
||
from scraper_kit.providers.yandex.serp import _parse_gate_json as new_parse_gate_json
|
||
from scraper_kit.providers.yandex.shared import normalize_house_type as new_normalize_house_type
|
||
from scraper_kit.providers.yandex.valuation import (
|
||
YandexValuationScraper as NewValuationScraper,
|
||
)
|
||
|
||
from app.services.scrapers.house_type_normalizer import (
|
||
normalize_house_type as old_normalize_house_type,
|
||
)
|
||
from app.services.scrapers.yandex_detail import YandexDetailScraper as OldDetailScraper
|
||
from app.services.scrapers.yandex_newbuilding import (
|
||
YandexNewbuildingScraper as OldNewbuildingScraper,
|
||
)
|
||
from app.services.scrapers.yandex_realty import YandexRealtyScraper as OldRealtyScraper
|
||
from app.services.scrapers.yandex_realty import _entity_to_lot as old_entity_to_lot
|
||
from app.services.scrapers.yandex_realty import _parse_gate_json as old_parse_gate_json
|
||
from app.services.scrapers.yandex_valuation import (
|
||
YandexValuationScraper as OldValuationScraper,
|
||
)
|
||
|
||
_FIXTURES = Path(__file__).parent / "fixtures"
|
||
|
||
# Инжектируемый конфиг для kit-скрапперов (duck-typed namespace, structurally
|
||
# удовлетворяет scraper_kit.contracts.ScraperConfig в объёме читаемых полей).
|
||
_KIT_CONFIG = SimpleNamespace(
|
||
scraper_proxy_url=None,
|
||
yandex_proxy_rotate_url=None,
|
||
avito_proxy_rotate_url=None,
|
||
)
|
||
|
||
|
||
# ── Fixtures: gate-API entities (mirror test_yandex_realty_serp) ─────────────
|
||
|
||
_ENTITY_FULL: dict = {
|
||
"offerId": 4740475460451078271,
|
||
"url": "//realty.yandex.ru/offer/4740475460451078271",
|
||
"price": {"currency": "RUR", "value": 4500000, "valuePerPart": 119048},
|
||
"area": {"value": 37.8, "unit": "SQUARE_METER"},
|
||
"livingSpace": {"value": 20, "unit": "SQUARE_METER"},
|
||
"kitchenSpace": {"value": 12.7, "unit": "SQUARE_METER"},
|
||
"roomsTotal": 1,
|
||
"floorsOffered": [8],
|
||
"floorsTotal": 25,
|
||
"ceilingHeight": 2.7,
|
||
"building": {
|
||
"builtYear": 2016,
|
||
"buildingType": "MONOLIT",
|
||
"siteId": 280938,
|
||
"siteName": "Peremen",
|
||
},
|
||
"location": {
|
||
"geocoderAddress": "Ekaterinburg, ulitsa Evgenia Savkova, 8",
|
||
"point": {"latitude": 56.79512, "longitude": 60.49186},
|
||
},
|
||
"mainImages": [
|
||
"//avatars.mds.yandex.net/get-realty-offers/11904018/abc/main",
|
||
"//avatars.mds.yandex.net/get-realty-offers/14717586/def/main",
|
||
],
|
||
}
|
||
|
||
_ENTITY_STUDIO: dict = {
|
||
"offerId": 9999999,
|
||
"url": "//realty.yandex.ru/offer/9999999",
|
||
"price": {"value": 3200000, "valuePerPart": 114285},
|
||
"area": {"value": 28.0},
|
||
"livingSpace": None,
|
||
"kitchenSpace": None,
|
||
"roomsTotal": None, # studio -> rooms=0
|
||
"floorsOffered": [2],
|
||
"floorsTotal": 9,
|
||
"ceilingHeight": None,
|
||
"building": {"builtYear": 2010, "buildingType": "PANEL", "siteId": None, "siteName": None},
|
||
"location": {
|
||
"geocoderAddress": "Ekaterinburg, ulitsa Lenina, 5",
|
||
"point": {"latitude": 56.838, "longitude": 60.597},
|
||
},
|
||
"mainImages": [],
|
||
}
|
||
|
||
_ENTITY_RICH_OWNER: dict = {
|
||
"offerId": 5500000000000000001,
|
||
"url": "//realty.yandex.ru/offer/5500000000000000001",
|
||
"creationDate": "2026-05-24T15:10:27Z",
|
||
"description": "Продаётся светлая квартира с видом на парк.",
|
||
"author": {"category": "OWNER", "agentName": "Иван", "organization": None},
|
||
"price": {"value": 7200000, "valuePerPart": 150000, "trend": "DECREASED", "previous": 7500000},
|
||
"predictions": {"predictedPrice": {"value": "7554000", "min": "7100000", "max": "8000000"}},
|
||
"area": {"value": 48.0},
|
||
"roomsTotal": 2,
|
||
"floorsOffered": [5],
|
||
"floorsTotal": 12,
|
||
"building": {"builtYear": 2018, "buildingType": "MONOLIT"},
|
||
"location": {
|
||
"geocoderAddress": "Ekaterinburg, ulitsa Mira, 10",
|
||
"point": {"latitude": 56.84, "longitude": 60.61},
|
||
"metroList": [
|
||
{"name": "Ploshchad 1905 goda", "timeToMetro": 7},
|
||
{"name": "Geologicheskaya", "timeToMetro": 12},
|
||
],
|
||
},
|
||
"mainImages": [],
|
||
}
|
||
|
||
_ENTITY_RICH_AGENCY: dict = {
|
||
"offerId": 5500000000000000002,
|
||
"url": "//realty.yandex.ru/offer/5500000000000000002",
|
||
"creationDate": "2026-06-01T09:00:00Z",
|
||
"description": "Агентская продажа.",
|
||
"author": {"category": "AGENCY", "agentName": "Петров", "organization": "Агентство «Диал»"},
|
||
"price": {"value": 9900000, "trend": "UNCHANGED"},
|
||
"area": {"value": 60.0},
|
||
"roomsTotal": 3,
|
||
"floorsOffered": [3],
|
||
"floorsTotal": 9,
|
||
"building": {"builtYear": 2012, "buildingType": "BRICK"},
|
||
"location": {
|
||
"geocoderAddress": "Ekaterinburg, ulitsa Lenina, 1",
|
||
"point": {"latitude": 56.83, "longitude": 60.6},
|
||
"metro": {"name": "Dinamo", "timeToMetro": 4},
|
||
},
|
||
"mainImages": [],
|
||
}
|
||
|
||
_ALL_ENTITIES = [_ENTITY_FULL, _ENTITY_STUDIO, _ENTITY_RICH_OWNER, _ENTITY_RICH_AGENCY]
|
||
|
||
|
||
def _make_gate_payload(entities: list, pager: dict | None = None) -> dict:
|
||
if pager is None:
|
||
pager = {"page": 0, "pageSize": 20, "totalItems": len(entities), "totalPages": 1}
|
||
return {"response": {"search": {"offers": {"entities": entities, "pager": pager}}}}
|
||
|
||
|
||
def _dump_lots(lots: list[Any]) -> list[dict[str, Any]]:
|
||
return [lot.model_dump() for lot in lots]
|
||
|
||
|
||
# ── SERP _entity_to_lot parity ───────────────────────────────────────────────
|
||
|
||
|
||
def test_entity_to_lot_parity_all_entities() -> None:
|
||
"""gate-API entity → ScrapedLot идентичен на всех репрезентативных entity."""
|
||
for entity in _ALL_ENTITIES:
|
||
old_lot = old_entity_to_lot(entity)
|
||
new_lot = new_entity_to_lot(entity)
|
||
assert (old_lot is None) == (new_lot is None)
|
||
assert old_lot is not None and new_lot is not None
|
||
assert old_lot.model_dump() == new_lot.model_dump()
|
||
|
||
|
||
def test_entity_to_lot_parity_new_flat_segments() -> None:
|
||
"""new_flat=YES/NO → listing_segment идентичен (новостройка/вторичка)."""
|
||
for new_flat in ("NO", "YES"):
|
||
old_lot = old_entity_to_lot(_ENTITY_FULL, new_flat=new_flat)
|
||
new_lot = new_entity_to_lot(_ENTITY_FULL, new_flat=new_flat)
|
||
assert old_lot is not None and new_lot is not None
|
||
assert old_lot.model_dump() == new_lot.model_dump()
|
||
|
||
|
||
def test_entity_to_lot_parity_ceiling_out_of_range() -> None:
|
||
"""ceilingHeight мусор (18 / 1.6) → одинаково дропается в None (#2007)."""
|
||
for bad_ceiling in (18, 1.6):
|
||
entity = {**_ENTITY_FULL, "ceilingHeight": bad_ceiling}
|
||
old_lot = old_entity_to_lot(entity)
|
||
new_lot = new_entity_to_lot(entity)
|
||
assert old_lot is not None and new_lot is not None
|
||
assert old_lot.model_dump() == new_lot.model_dump()
|
||
|
||
|
||
# ── SERP _parse_gate_json parity ─────────────────────────────────────────────
|
||
|
||
|
||
def test_parse_gate_json_parity() -> None:
|
||
"""payload → list[ScrapedLot] идентичен (вторичка и новостройка)."""
|
||
payload = _make_gate_payload(_ALL_ENTITIES)
|
||
for new_flat in ("NO", "YES"):
|
||
old_lots = old_parse_gate_json(payload, page_param=2, new_flat=new_flat)
|
||
new_lots = new_parse_gate_json(payload, page_param=2, new_flat=new_flat)
|
||
assert len(old_lots) == len(new_lots) > 0
|
||
assert _dump_lots(old_lots) == _dump_lots(new_lots)
|
||
|
||
|
||
# ── normalize_house_type parity ──────────────────────────────────────────────
|
||
|
||
|
||
def test_normalize_house_type_parity() -> None:
|
||
"""SCREAMING / camelCase / канон / мусор → идентичный результат."""
|
||
raws = [
|
||
None,
|
||
"",
|
||
"MONOLIT",
|
||
"BRICK",
|
||
"PANEL",
|
||
"MONOLIT_BRICK",
|
||
"monolithBrick",
|
||
"gasSilicateBlock",
|
||
"monolith_brick",
|
||
"monolith",
|
||
"stalin",
|
||
"OTHER",
|
||
" MONOLIT ",
|
||
]
|
||
for raw in raws:
|
||
assert old_normalize_house_type(raw) == new_normalize_house_type(raw)
|
||
|
||
|
||
# ── detail parse parity (реальные offer-HTML фикстуры) ───────────────────────
|
||
|
||
_SECONDARY_ID = "3402418396407468801"
|
||
_STUDIO_ID = "7811691822126445498"
|
||
|
||
|
||
def _load_offer(offer_id: str) -> str:
|
||
return (_FIXTURES / f"yandex_offer_{offer_id}.html").read_text(encoding="utf-8")
|
||
|
||
|
||
def _offer_url(offer_id: str) -> str:
|
||
return f"https://realty.yandex.ru/offer/{offer_id}/"
|
||
|
||
|
||
def test_detail_parse_parity() -> None:
|
||
"""INITIAL_STATE.offerCard.card + JSON-LD → DetailEnrichment идентичен."""
|
||
old_scraper = OldDetailScraper()
|
||
new_scraper = NewDetailScraper()
|
||
for offer_id in (_SECONDARY_ID, _STUDIO_ID):
|
||
html = _load_offer(offer_id)
|
||
url = _offer_url(offer_id)
|
||
old_res = old_scraper.parse(html, offer_url=url)
|
||
new_res = new_scraper.parse(html, offer_url=url)
|
||
assert old_res is not None and new_res is not None
|
||
assert old_res.model_dump() == new_res.model_dump()
|
||
|
||
|
||
# ── valuation parse parity (синтетика) ───────────────────────────────────────
|
||
|
||
_VALUATION_HTML = """
|
||
<html><body>
|
||
Дом 2016 года 25 этажей 2,7 м потолки Лифт 42 объекта Панорама
|
||
48,5 м², 1-комнатная, 5 этаж 5,1 млн ₽ 105 155 ₽ за м² 23.10.2023
|
||
В экспозиции 945 дней Снято 24.05.2024
|
||
32 м², студия, 2 этаж 3,2 млн ₽ 100 000 ₽ за м² 01.02.2024 В продаже
|
||
</body></html>
|
||
"""
|
||
|
||
|
||
def test_valuation_parse_parity() -> None:
|
||
"""house-meta + history (chunked-text fallback) → идентичный результат."""
|
||
old_scraper = OldValuationScraper()
|
||
new_scraper = NewValuationScraper(config=_KIT_CONFIG)
|
||
kwargs = dict(
|
||
address="Екатеринбург, ул. Ленина, 5",
|
||
offer_category="APARTMENT",
|
||
offer_type="SELL",
|
||
page=1,
|
||
source_url="https://realty.yandex.ru/otsenka-kvartiry-po-adresu-onlayn/?address=test",
|
||
)
|
||
old_res = old_scraper.parse(_VALUATION_HTML, **kwargs)
|
||
new_res = new_scraper.parse(_VALUATION_HTML, **kwargs)
|
||
assert old_res.model_dump() == new_res.model_dump()
|
||
|
||
|
||
# ── newbuilding parse parity (синтетика) ─────────────────────────────────────
|
||
|
||
_NEWBUILDING_HTML = """
|
||
<html><body>
|
||
<h1>ЖК «Татлин»</h1>
|
||
Екатеринбург, ул. Черепанова 4.3 из 5 1505 оценок Смотреть все 353 отзыва
|
||
56.855312 60.576668
|
||
<h2>О комплексе</h2>
|
||
<div>Дом комфорт-класса, монолитный. Введён в эксплуатацию в июнь 2023.
|
||
35-этажные башни на участке 1,5 га. Три корпуса.</div>
|
||
</body></html>
|
||
"""
|
||
|
||
|
||
def test_newbuilding_parse_parity() -> None:
|
||
"""ЖК-лендинг → YandexNewbuildingInfo идентичен."""
|
||
old_scraper = OldNewbuildingScraper()
|
||
new_scraper = NewNewbuildingScraper()
|
||
old_res = old_scraper.parse(
|
||
_NEWBUILDING_HTML,
|
||
jk_slug="tatlin",
|
||
jk_id="1592987",
|
||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||
)
|
||
new_res = new_scraper.parse(
|
||
_NEWBUILDING_HTML,
|
||
jk_slug="tatlin",
|
||
jk_id="1592987",
|
||
source_url="https://realty.yandex.ru/ekaterinburg/kupit/novostrojka/tatlin-1592987/",
|
||
)
|
||
assert old_res.model_dump() == new_res.model_dump()
|
||
|
||
|
||
# ── _build_url parity ────────────────────────────────────────────────────────
|
||
|
||
|
||
def test_build_url_parity() -> None:
|
||
"""gate-API URL builder идентичен на репрезентативных комбо."""
|
||
old_scraper = OldRealtyScraper()
|
||
new_scraper = NewRealtyScraper(config=_KIT_CONFIG)
|
||
combos = [
|
||
dict(page=1),
|
||
dict(page=3, rooms="2", price_min=5_000_000, price_max=7_000_000),
|
||
dict(page=1, rooms="studio", new_flat="YES"),
|
||
dict(page=2, rooms="4+", price_min=25_000_000),
|
||
]
|
||
for combo in combos:
|
||
assert old_scraper._build_url(**combo) == new_scraper._build_url(**combo)
|
||
|
||
|
||
# ── strangler-guard: kit-провайдер не импортирует app.* ──────────────────────
|
||
|
||
|
||
def test_kit_yandex_has_no_app_imports() -> None:
|
||
"""Ни один модуль scraper_kit.providers.yandex.* не импортирует app.* (развязка)."""
|
||
from scraper_kit.providers.yandex import detail, newbuilding, serp, shared, valuation
|
||
|
||
for mod in (serp, detail, valuation, newbuilding, shared):
|
||
source = inspect.getsource(mod)
|
||
for line in source.splitlines():
|
||
stripped = line.strip()
|
||
assert not stripped.startswith("from app"), f"{mod.__name__}: {line!r}"
|
||
assert not stripped.startswith("import app"), f"{mod.__name__}: {line!r}"
|