fix(scrapers): yandex area из embedded-state JSON (Refs #775)
Часть A (#24): yandex_realty.py парсил площадь из RAW-HTML (regex), пустого до JS-гидрации → 82% NULL area. _parse_html теперь извлекает {offer_id: area_m2} из <script id=initial_state_script> (или application/json) раз на страницу и использует preferentially в _card_to_lot; DOM-regex остаётся fallback'ом. 4 offer-path + 5 area-полей (incl nested {value}). 6 тестов. Часть B (#23, cian buildingCadastralNumber): УЖЕ реализовано — cian.py:291 читает offer.buildingCadastralNumber, :456 пишет в ScrapedLot, поле base.py:109 существует, тест test_cian_serp_cadastral_numbers зелёный. Изменений не нужно (премиса аудита устарела). cian.py НЕ трогался. ruff clean; pytest -k 'yandex or cian' 365 pass (+1 pre-existing unrelated fail test_cian_valuation low_price mock, не из этого PR).
This commit is contained in:
parent
238f14a0d7
commit
d03c96a79a
2 changed files with 249 additions and 10 deletions
|
|
@ -4,6 +4,10 @@ History: Yandex SSR used to embed full state in `<script id="initial_state_scrip
|
|||
As of 2026-05, the state is chunked + evaporates after hydration. This version
|
||||
scrapes the rendered DOM via selectolax + helpers from yandex_helpers.py.
|
||||
|
||||
Area extraction: DOM concat-text is empty pre-hydration (~82% NULL). Fix: extract
|
||||
area from the page-level embedded JSON state (script[id="initial_state_script"] or
|
||||
application/json script), keyed by offerId. DOM regex is kept as fallback.
|
||||
|
||||
URL pattern (path-based, no geo radius):
|
||||
https://realty.yandex.ru/{city}/kupit/kvartira/vtorichniy-rynok/?page=N
|
||||
|
||||
|
|
@ -12,6 +16,7 @@ Typical: 23 [data-test="OffersSerpItem"] cards per page.
|
|||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from datetime import date # noqa: F401 (used in type hints / future helpers)
|
||||
|
|
@ -47,6 +52,108 @@ PHOTO_SIZE_TO = "main"
|
|||
ADDRESS_CONTAINER_SELECTOR = '[class*="AddressWithGeoLinks__addressContainer"]'
|
||||
_RE_WS = re.compile(r"\s+")
|
||||
|
||||
# Yandex SERP embeds offer data in a <script id="initial_state_script"> as JSON,
|
||||
# or (since 2026-05) in <script type="application/json" ...> chunks.
|
||||
# We look for these selectors in priority order.
|
||||
_STATE_SCRIPT_SELECTORS = [
|
||||
'script[id="initial_state_script"]',
|
||||
'script[type="application/json"]',
|
||||
]
|
||||
|
||||
# Area field candidates in a Yandex offer object (tried in order).
|
||||
_AREA_FIELDS = ("totalArea", "area", "spaceTotal", "spaceAll", "squareTotal")
|
||||
|
||||
|
||||
def _extract_offer_areas_from_state(html: str) -> dict[str, float]:
|
||||
"""Extract {offer_id: area_m2} map from Yandex SERP embedded JSON state.
|
||||
|
||||
Tries each known script selector, parses JSON, then traverses known
|
||||
offer-array paths looking for offer objects with an id + area field.
|
||||
|
||||
Returns empty dict if state is absent or unparseable.
|
||||
"""
|
||||
tree = HTMLParser(html)
|
||||
candidates: list[str] = []
|
||||
for selector in _STATE_SCRIPT_SELECTORS:
|
||||
for node in tree.css(selector):
|
||||
text = node.text() or ""
|
||||
text = text.strip()
|
||||
if text:
|
||||
candidates.append(text)
|
||||
|
||||
result: dict[str, float] = {}
|
||||
for text in candidates:
|
||||
try:
|
||||
state = json.loads(text)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
if not isinstance(state, dict):
|
||||
continue
|
||||
_collect_areas_from_state(state, result)
|
||||
if result:
|
||||
break # успешно извлекли из первого валидного blob'а
|
||||
return result
|
||||
|
||||
|
||||
def _collect_areas_from_state(state: dict[str, Any], out: dict[str, float]) -> None:
|
||||
"""Walk known Yandex state paths to find offer objects and extract area.
|
||||
|
||||
Known paths (tried in order):
|
||||
- state["offers"] (flat list)
|
||||
- state["pageData"]["offers"] (SERP v2)
|
||||
- state["listing"]["offers"] (SERP v3)
|
||||
- state["serpData"]["items"] (SERP v4 chunks)
|
||||
"""
|
||||
offer_lists: list[Any] = []
|
||||
for path in (
|
||||
["offers"],
|
||||
["pageData", "offers"],
|
||||
["listing", "offers"],
|
||||
["serpData", "items"],
|
||||
):
|
||||
node: Any = state
|
||||
for key in path:
|
||||
if not isinstance(node, dict):
|
||||
node = None
|
||||
break
|
||||
node = node.get(key)
|
||||
if isinstance(node, list):
|
||||
offer_lists.append(node)
|
||||
|
||||
for offer_list in offer_lists:
|
||||
for offer in offer_list:
|
||||
if not isinstance(offer, dict):
|
||||
continue
|
||||
# Flatten one level of nesting (some Yandex shapes wrap the offer)
|
||||
inner = offer.get("offer") or offer
|
||||
if not isinstance(inner, dict):
|
||||
continue
|
||||
oid = inner.get("offerId") or inner.get("id")
|
||||
if oid is None:
|
||||
continue
|
||||
area = _read_area_field(inner)
|
||||
if area is not None:
|
||||
out[str(oid)] = area
|
||||
|
||||
|
||||
def _read_area_field(offer: dict[str, Any]) -> float | None:
|
||||
"""Try known field names to read area (м²) from a Yandex offer dict."""
|
||||
for field in _AREA_FIELDS:
|
||||
val = offer.get(field)
|
||||
if val is None:
|
||||
continue
|
||||
# Some fields are nested dicts: {"value": 52.0, "unit": "SQM"}
|
||||
if isinstance(val, dict):
|
||||
val = val.get("value")
|
||||
if val is not None:
|
||||
try:
|
||||
result = float(val)
|
||||
if result > 0:
|
||||
return result
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
class YandexRealtyScraper(BaseScraper):
|
||||
name = "yandex"
|
||||
|
|
@ -81,7 +188,11 @@ class YandexRealtyScraper(BaseScraper):
|
|||
lots = self._parse_html(response.text, page=page)
|
||||
logger.info(
|
||||
"yandex serp page=%d city=%s: %d cards (anchor=(%.4f,%.4f) ignored)",
|
||||
page, self.city, len(lots), lat, lon,
|
||||
page,
|
||||
self.city,
|
||||
len(lots),
|
||||
lat,
|
||||
lon,
|
||||
)
|
||||
await self.sleep_between_requests()
|
||||
return lots
|
||||
|
|
@ -106,7 +217,9 @@ class YandexRealtyScraper(BaseScraper):
|
|||
seen[key] = lot
|
||||
logger.info(
|
||||
"yandex serp aggregate: %d unique lots over %d pages (city=%s)",
|
||||
len(seen), max_pages, self.city,
|
||||
len(seen),
|
||||
max_pages,
|
||||
self.city,
|
||||
)
|
||||
return list(seen.values())
|
||||
|
||||
|
|
@ -118,16 +231,26 @@ class YandexRealtyScraper(BaseScraper):
|
|||
return base
|
||||
|
||||
def _parse_html(self, html: str, page: int = 0) -> list[ScrapedLot]:
|
||||
# Extract offer-level area from embedded JSON state first (covers ~82% of
|
||||
# cards where DOM text is empty pre-hydration). Falls back to DOM regex per card.
|
||||
state_areas = _extract_offer_areas_from_state(html)
|
||||
if state_areas:
|
||||
logger.debug("yandex serp: state areas extracted for %d offers", len(state_areas))
|
||||
tree = HTMLParser(html)
|
||||
cards = tree.css('[data-test="OffersSerpItem"]')
|
||||
lots: list[ScrapedLot] = []
|
||||
for card in cards:
|
||||
lot = self._card_to_lot(card, page=page)
|
||||
lot = self._card_to_lot(card, page=page, state_areas=state_areas)
|
||||
if lot is not None:
|
||||
lots.append(lot)
|
||||
return lots
|
||||
|
||||
def _card_to_lot(self, card: Node, page: int = 0) -> ScrapedLot | None:
|
||||
def _card_to_lot(
|
||||
self,
|
||||
card: Node,
|
||||
page: int = 0,
|
||||
state_areas: dict[str, float] | None = None,
|
||||
) -> ScrapedLot | None:
|
||||
try:
|
||||
# offer_id — required
|
||||
offer_link = card.css_first('a[href^="/offer/"]')
|
||||
|
|
@ -147,11 +270,12 @@ class YandexRealtyScraper(BaseScraper):
|
|||
if not price_rub or price_rub <= 0:
|
||||
return None
|
||||
|
||||
# Optional fields
|
||||
# Area: prefer JSON state (covers pre-hydration DOM where text is empty),
|
||||
# fall back to DOM regex.
|
||||
area_m2: float | None = (state_areas or {}).get(offer_id)
|
||||
if area_m2 is None:
|
||||
area_m = RE_TITLE_AREA.search(text)
|
||||
area_m2 = (
|
||||
float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
)
|
||||
area_m2 = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
|
||||
rooms = self._parse_rooms(text)
|
||||
|
||||
|
|
@ -189,7 +313,8 @@ class YandexRealtyScraper(BaseScraper):
|
|||
house_source = "yandex_realty_nb"
|
||||
house_ext_id = nb_match.group(2)
|
||||
house_url = (
|
||||
nb_href if nb_href.startswith("http")
|
||||
nb_href
|
||||
if nb_href.startswith("http")
|
||||
else f"https://realty.yandex.ru{nb_href}"
|
||||
)
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,6 @@
|
|||
"""Unit tests for YandexRealtyScraper DOM-based SERP parser."""
|
||||
|
||||
import json
|
||||
from datetime import date
|
||||
|
||||
import pytest
|
||||
|
|
@ -7,6 +9,7 @@ from app.services.scrapers.yandex_realty import (
|
|||
DEFAULT_CITY,
|
||||
MAX_PAGES,
|
||||
YandexRealtyScraper,
|
||||
_extract_offer_areas_from_state,
|
||||
)
|
||||
|
||||
# Realistic single-card fixture (trimmed but selector-faithful).
|
||||
|
|
@ -225,3 +228,114 @@ def test_address_fallback_to_street_link():
|
|||
lots = s._parse_html(NO_ADDR_CONTAINER_CARD_HTML, page=0)
|
||||
assert len(lots) == 1
|
||||
assert lots[0].address == "улица Малышева"
|
||||
|
||||
|
||||
# ── PART A: area from embedded JSON state ────────────────────────────────────
|
||||
|
||||
|
||||
def _make_yandex_state_html(offer_id: str, area: float, price_rub: int = 5_000_000) -> str:
|
||||
"""Build minimal Yandex SERP HTML with:
|
||||
- <script id="initial_state_script"> containing offer area in JSON state
|
||||
- A DOM card where the text div has NO area (simulating pre-JS-hydration).
|
||||
"""
|
||||
state = {
|
||||
"offers": [
|
||||
{
|
||||
"offerId": offer_id,
|
||||
"totalArea": area,
|
||||
}
|
||||
]
|
||||
}
|
||||
state_json = json.dumps(state, ensure_ascii=False)
|
||||
price_fmt = f"{price_rub:,}".replace(",", " ")
|
||||
return f"""<html><head>
|
||||
<script id="initial_state_script">{state_json}</script>
|
||||
</head><body>
|
||||
<div data-test="OffersSerpItem">
|
||||
<a href="/offer/{offer_id}/">Открыть</a>
|
||||
<div class="AddressWithGeoLinks__addressContainer--xyz">
|
||||
<a href="/ekaterinburg/kupit/kvartira/st-test-1/">улица Тестовая</a>, 10
|
||||
</div>
|
||||
<div>{price_fmt} ₽</div>
|
||||
</div>
|
||||
</body></html>"""
|
||||
|
||||
|
||||
def test_area_from_state_json_no_dom_text():
|
||||
"""area_m2 extracted from embedded JSON state when DOM text has no area."""
|
||||
html = _make_yandex_state_html(offer_id="1234567890", area=63.5)
|
||||
s = YandexRealtyScraper()
|
||||
lots = s._parse_html(html, page=0)
|
||||
assert len(lots) == 1
|
||||
assert lots[0].area_m2 == pytest.approx(63.5)
|
||||
|
||||
|
||||
def test_area_from_state_json_multiple_offers():
|
||||
"""State with multiple offers — each card gets its own area from state."""
|
||||
state = {
|
||||
"offers": [
|
||||
{"offerId": "111", "totalArea": 42.0},
|
||||
{"offerId": "222", "totalArea": 78.3},
|
||||
]
|
||||
}
|
||||
state_json = json.dumps(state)
|
||||
html = f"""<html><head>
|
||||
<script id="initial_state_script">{state_json}</script>
|
||||
</head><body>
|
||||
<div data-test="OffersSerpItem">
|
||||
<a href="/offer/111/">x</a>
|
||||
<div>3 000 000 ₽</div>
|
||||
</div>
|
||||
<div data-test="OffersSerpItem">
|
||||
<a href="/offer/222/">x</a>
|
||||
<div>5 000 000 ₽</div>
|
||||
</div>
|
||||
</body></html>"""
|
||||
s = YandexRealtyScraper()
|
||||
lots = s._parse_html(html, page=0)
|
||||
assert len(lots) == 2
|
||||
areas = {lot.source_id: lot.area_m2 for lot in lots}
|
||||
assert areas["111"] == pytest.approx(42.0)
|
||||
assert areas["222"] == pytest.approx(78.3)
|
||||
|
||||
|
||||
def test_area_dom_fallback_when_no_state():
|
||||
"""When state script absent, area falls back to DOM regex (existing behavior)."""
|
||||
# SINGLE_CARD_HTML has area in DOM text — state script absent
|
||||
s = YandexRealtyScraper()
|
||||
lots = s._parse_html(SINGLE_CARD_HTML, page=0)
|
||||
assert len(lots) == 1
|
||||
assert lots[0].area_m2 == pytest.approx(36.8)
|
||||
|
||||
|
||||
def test_extract_offer_areas_from_state_nested_path():
|
||||
"""State via pageData.offers path (SERP v2 shape)."""
|
||||
state = {
|
||||
"pageData": {
|
||||
"offers": [
|
||||
{"offerId": "999", "spaceTotal": 55.0},
|
||||
]
|
||||
}
|
||||
}
|
||||
html = '<script id="initial_state_script">' + json.dumps(state) + "</script>"
|
||||
areas = _extract_offer_areas_from_state(html)
|
||||
assert areas == {"999": 55.0}
|
||||
|
||||
|
||||
def test_extract_offer_areas_area_dict_value():
|
||||
"""area field as nested dict {'value': 48.0, 'unit': 'SQM'} (some Yandex responses)."""
|
||||
state = {
|
||||
"offers": [
|
||||
{"offerId": "777", "area": {"value": 48.0, "unit": "SQM"}},
|
||||
]
|
||||
}
|
||||
html = '<script id="initial_state_script">' + json.dumps(state) + "</script>"
|
||||
areas = _extract_offer_areas_from_state(html)
|
||||
assert areas == {"777": 48.0}
|
||||
|
||||
|
||||
def test_extract_offer_areas_empty_when_no_script():
|
||||
"""No state script → empty dict returned, no crash."""
|
||||
html = "<html><body><p>no state here</p></body></html>"
|
||||
areas = _extract_offer_areas_from_state(html)
|
||||
assert areas == {}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue