243 lines
8 KiB
Python
243 lines
8 KiB
Python
"""Yandex Realty parsing helpers — JSON-LD + regex + DOM utilities for SERP, detail, newbuilding,
|
|
and valuation parsers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import re
|
|
from datetime import date
|
|
from typing import Any
|
|
|
|
from selectolax.parser import HTMLParser
|
|
|
|
__all__ = [
|
|
"RE_AGENCY_FOUNDED",
|
|
"RE_AGENCY_OBJECTS",
|
|
"RE_FLOOR",
|
|
"RE_JK_ID",
|
|
"RE_METRO_WALK",
|
|
"RE_OFFER_ID",
|
|
"RE_PPM2",
|
|
"RE_PRICE",
|
|
"RE_STREET_ID",
|
|
"RE_TITLE_AREA",
|
|
"RE_TITLE_ROOMS",
|
|
"RE_VIEWS",
|
|
"RE_YEAR",
|
|
"RU_MONTHS",
|
|
"RU_MONTH_NOMINATIVE",
|
|
"extract_json_ld",
|
|
"find_ld_by_type",
|
|
"parse_dmy",
|
|
"parse_house_class",
|
|
"parse_house_type",
|
|
"parse_ru_date",
|
|
"parse_rub",
|
|
]
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Section 1 — JSON-LD extraction
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def extract_json_ld(html: str) -> list[dict[str, Any]]:
|
|
"""Extract all <script type='application/ld+json'> blocks from HTML.
|
|
|
|
Returns a list of parsed JSON-LD dicts. Skips blocks that fail to parse.
|
|
"""
|
|
tree = HTMLParser(html)
|
|
results: list[dict[str, Any]] = []
|
|
for script in tree.css('script[type="application/ld+json"]'):
|
|
text = script.text() or ""
|
|
if not text.strip():
|
|
continue
|
|
try:
|
|
parsed = json.loads(text)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if isinstance(parsed, dict):
|
|
results.append(parsed)
|
|
elif isinstance(parsed, list):
|
|
# Some pages wrap multiple LDs in a JSON array
|
|
results.extend(item for item in parsed if isinstance(item, dict))
|
|
return results
|
|
|
|
|
|
def find_ld_by_type(html: str, type_name: str) -> dict[str, Any] | None:
|
|
"""Return the first JSON-LD dict whose @type equals type_name.
|
|
|
|
If @type is a list, returns the dict if type_name is contained in it.
|
|
"""
|
|
for ld in extract_json_ld(html):
|
|
t = ld.get("@type")
|
|
if t == type_name:
|
|
return ld
|
|
if isinstance(t, list) and type_name in t:
|
|
return ld
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Section 2 — Regex extractors (module-level compiled)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# URL pattern extractors
|
|
RE_OFFER_ID = re.compile(r"/offer/(\d+)/?")
|
|
RE_JK_ID = re.compile(r"/novostrojka/([\w-]+?)-(\d+)/?") # group(1)=slug, group(2)=id
|
|
RE_STREET_ID = re.compile(r"/kupit/kvartira/st-([\w-]+?)-(\d+)/?")
|
|
|
|
# Card / title text extractors
|
|
RE_TITLE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м²")
|
|
RE_TITLE_ROOMS = re.compile(r"(\d+)\s*-?\s*комнатн|(студи[яюй])", re.IGNORECASE)
|
|
RE_FLOOR = re.compile(r"(\d+)\s+этаж\s+из\s+(\d+)", re.IGNORECASE)
|
|
RE_PRICE = re.compile(r"(\d[\d\s]+\d)\s*₽")
|
|
RE_PPM2 = re.compile(r"(\d[\d\s]+)\s*₽\s*за\s*м²", re.IGNORECASE)
|
|
RE_VIEWS = re.compile(r"(\d+)\s+просмотр", re.IGNORECASE)
|
|
|
|
# Detail / NLP extractors
|
|
RE_METRO_WALK = re.compile(
|
|
r"(\d+)\s+минут\s+(?:неспешной\s+прогулки|ходьбы|пешком)", re.IGNORECASE
|
|
)
|
|
RE_YEAR = re.compile(r"\b(19\d{2}|20[0-2]\d)\b")
|
|
|
|
# Agency block ("Год основания 1995", "150 объектов")
|
|
RE_AGENCY_FOUNDED = re.compile(r"Год\s+основания\s+(\d{4})", re.IGNORECASE)
|
|
RE_AGENCY_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE)
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Section 3 — Russian date parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
RU_MONTHS: dict[str, int] = {
|
|
"января": 1, "февраля": 2, "марта": 3, "апреля": 4, "мая": 5, "июня": 6,
|
|
"июля": 7, "августа": 8, "сентября": 9, "октября": 10, "ноября": 11, "декабря": 12,
|
|
}
|
|
RU_MONTH_NOMINATIVE: dict[str, int] = {
|
|
"январь": 1, "февраль": 2, "март": 3, "апрель": 4, "май": 5, "июнь": 6,
|
|
"июль": 7, "август": 8, "сентябрь": 9, "октябрь": 10, "ноябрь": 11, "декабрь": 12,
|
|
}
|
|
RE_RU_DATE = re.compile(r"(\d{1,2})\s+(\w+)\s+(\d{4})")
|
|
RE_DMY = re.compile(r"(\d{2})\.(\d{2})\.(\d{4})")
|
|
|
|
|
|
def parse_ru_date(text: str | None) -> date | None:
|
|
"""Parse Russian date string like '9 мая 2026' → date(2026, 5, 9). Genitive month form only."""
|
|
if not text:
|
|
return None
|
|
m = RE_RU_DATE.search(text)
|
|
if not m:
|
|
return None
|
|
day_s, month_ru, year_s = m.groups()
|
|
month = RU_MONTHS.get(month_ru.lower())
|
|
if not month:
|
|
return None
|
|
try:
|
|
return date(int(year_s), month, int(day_s))
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
def parse_dmy(text: str | None) -> date | None:
|
|
"""Parse DD.MM.YYYY format (used by Yandex Valuation tool history) → date."""
|
|
if not text:
|
|
return None
|
|
m = RE_DMY.search(text)
|
|
if not m:
|
|
return None
|
|
try:
|
|
return date(int(m.group(3)), int(m.group(2)), int(m.group(1)))
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Section 4 — House type / class NLP from description text
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def parse_house_type(text: str | None) -> str | None:
|
|
"""Extract house material type from free-form description.
|
|
|
|
Returns: 'panel' | 'monolith_brick' | 'monolith' | 'brick' | 'block' | 'wood' | None.
|
|
Order matters: 'monolith_brick' is checked before 'monolith' and 'brick' alone.
|
|
"""
|
|
if not text:
|
|
return None
|
|
t = text.lower()
|
|
if "панельн" in t:
|
|
return "panel"
|
|
if "монолит" in t and "кирпич" in t:
|
|
return "monolith_brick"
|
|
if "монолит" in t:
|
|
return "monolith"
|
|
if "кирпичн" in t:
|
|
return "brick"
|
|
if "блочн" in t:
|
|
return "block"
|
|
if "деревянн" in t:
|
|
return "wood"
|
|
return None
|
|
|
|
|
|
def parse_house_class(text: str | None) -> str | None:
|
|
"""Extract ЖК class from description.
|
|
|
|
Returns: 'elite' | 'premium' | 'business' | 'comfort_plus' | 'comfort' | 'economy' | None.
|
|
"""
|
|
if not text:
|
|
return None
|
|
t = text.lower()
|
|
if re.search(r"класса?\s+элит|элитный\s+класс", t):
|
|
return "elite"
|
|
if re.search(r"класса?\s+премиум|премиум.{0,5}класс", t):
|
|
return "premium"
|
|
if re.search(r"класса?\s+бизнес", t):
|
|
return "business"
|
|
if re.search(r"класса?\s+комфорт\s*\+", t):
|
|
return "comfort_plus"
|
|
if re.search(r"класса?\s+комфорт(?!\s*\+)", t):
|
|
return "comfort"
|
|
if re.search(r"класса?\s+эконом", t):
|
|
return "economy"
|
|
return None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Section 5 — Money parsing
|
|
# ---------------------------------------------------------------------------
|
|
|
|
RE_RUB_MLN = re.compile(r"([\d,.]+)\s*млн\s*₽", re.IGNORECASE)
|
|
RE_RUB_RAW = re.compile(r"(\d[\d\s]*\d|\d)")
|
|
|
|
|
|
def parse_rub(text: str | None) -> int | None:
|
|
"""Parse Russian price string → integer rubles.
|
|
|
|
Supports:
|
|
'4 399 000 ₽' → 4_399_000
|
|
'4,4 млн ₽' → 4_400_000
|
|
'4.4 млн ₽' → 4_400_000
|
|
'117 500' → 117_500 (raw digits without currency, e.g. ppm2)
|
|
None / 'без цены' / '' → None
|
|
"""
|
|
if not text:
|
|
return None
|
|
text = text.strip()
|
|
if not text:
|
|
return None
|
|
|
|
m = RE_RUB_MLN.search(text)
|
|
if m:
|
|
try:
|
|
return int(float(m.group(1).replace(",", ".").replace(" ", "")) * 1_000_000)
|
|
except (ValueError, TypeError):
|
|
pass
|
|
|
|
m = RE_RUB_RAW.search(text)
|
|
if m:
|
|
cleaned = m.group(1).replace(" ", "")
|
|
try:
|
|
return int(cleaned)
|
|
except ValueError:
|
|
return None
|
|
return None
|