295 lines
9.5 KiB
Python
295 lines
9.5 KiB
Python
"""Yandex Realty parsing helpers — JSON-LD + regex + DOM utilities for SERP, detail, newbuilding,
|
||
and valuation parsers."""
|
||
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import re
|
||
from datetime import date, timedelta
|
||
from typing import Any
|
||
|
||
from selectolax.parser import HTMLParser
|
||
|
||
__all__ = [
|
||
"RE_AGENCY_FOUNDED",
|
||
"RE_AGENCY_OBJECTS",
|
||
"RE_FLOOR",
|
||
"RE_JK_ID",
|
||
"RE_METRO_WALK",
|
||
"RE_OFFER_ID",
|
||
"RE_PPM2",
|
||
"RE_PRICE",
|
||
"RE_STREET_ID",
|
||
"RE_TITLE_AREA",
|
||
"RE_TITLE_ROOMS",
|
||
"RE_VIEWS",
|
||
"RE_YEAR",
|
||
"RU_MONTHS",
|
||
"RU_MONTH_NOMINATIVE",
|
||
"extract_json_ld",
|
||
"find_ld_by_type",
|
||
"parse_dmy",
|
||
"parse_house_class",
|
||
"parse_house_type",
|
||
"parse_listing_date",
|
||
"parse_ru_date",
|
||
"parse_rub",
|
||
]
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Section 1 — JSON-LD extraction
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def extract_json_ld(html: str) -> list[dict[str, Any]]:
|
||
"""Extract all <script type='application/ld+json'> blocks from HTML.
|
||
|
||
Returns a list of parsed JSON-LD dicts. Skips blocks that fail to parse.
|
||
"""
|
||
tree = HTMLParser(html)
|
||
results: list[dict[str, Any]] = []
|
||
for script in tree.css('script[type="application/ld+json"]'):
|
||
text = script.text() or ""
|
||
if not text.strip():
|
||
continue
|
||
try:
|
||
parsed = json.loads(text)
|
||
except json.JSONDecodeError:
|
||
continue
|
||
if isinstance(parsed, dict):
|
||
results.append(parsed)
|
||
elif isinstance(parsed, list):
|
||
# Some pages wrap multiple LDs in a JSON array
|
||
results.extend(item for item in parsed if isinstance(item, dict))
|
||
return results
|
||
|
||
|
||
def find_ld_by_type(html: str, type_name: str) -> dict[str, Any] | None:
|
||
"""Return the first JSON-LD dict whose @type equals type_name.
|
||
|
||
If @type is a list, returns the dict if type_name is contained in it.
|
||
"""
|
||
for ld in extract_json_ld(html):
|
||
t = ld.get("@type")
|
||
if t == type_name:
|
||
return ld
|
||
if isinstance(t, list) and type_name in t:
|
||
return ld
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Section 2 — Regex extractors (module-level compiled)
|
||
# ---------------------------------------------------------------------------
|
||
|
||
# URL pattern extractors
|
||
RE_OFFER_ID = re.compile(r"/offer/(\d+)/?")
|
||
RE_JK_ID = re.compile(r"/novostrojka/([\w-]+?)-(\d+)/?") # group(1)=slug, group(2)=id
|
||
RE_STREET_ID = re.compile(r"/kupit/kvartira/st-([\w-]+?)-(\d+)/?")
|
||
|
||
# Card / title text extractors
|
||
RE_TITLE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м²")
|
||
RE_TITLE_ROOMS = re.compile(r"(\d+)\s*-?\s*комнатн|(студи[яюй])", re.IGNORECASE)
|
||
RE_FLOOR = re.compile(r"(\d+)\s+этаж\s+из\s+(\d+)", re.IGNORECASE)
|
||
RE_PRICE = re.compile(r"(\d[\d\s]+\d)\s*₽")
|
||
RE_PPM2 = re.compile(r"(\d[\d\s]+)\s*₽\s*за\s*м²", re.IGNORECASE)
|
||
RE_VIEWS = re.compile(r"(\d+)\s+просмотр", re.IGNORECASE)
|
||
|
||
# Detail / NLP extractors
|
||
RE_METRO_WALK = re.compile(
|
||
r"(\d+)\s+минут\s+(?:неспешной\s+прогулки|ходьбы|пешком)", re.IGNORECASE
|
||
)
|
||
RE_YEAR = re.compile(r"\b(19\d{2}|20[0-2]\d)\b")
|
||
|
||
# Agency block ("Год основания 1995", "150 объектов")
|
||
RE_AGENCY_FOUNDED = re.compile(r"Год\s+основания\s+(\d{4})", re.IGNORECASE)
|
||
RE_AGENCY_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE)
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Section 3 — Russian date parsing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
RU_MONTHS: dict[str, int] = {
|
||
"января": 1, "февраля": 2, "марта": 3, "апреля": 4, "мая": 5, "июня": 6,
|
||
"июля": 7, "августа": 8, "сентября": 9, "октября": 10, "ноября": 11, "декабря": 12,
|
||
}
|
||
RU_MONTH_NOMINATIVE: dict[str, int] = {
|
||
"январь": 1, "февраль": 2, "март": 3, "апрель": 4, "май": 5, "июнь": 6,
|
||
"июль": 7, "август": 8, "сентябрь": 9, "октябрь": 10, "ноябрь": 11, "декабрь": 12,
|
||
}
|
||
RE_RU_DATE = re.compile(r"(\d{1,2})\s+(\w+)\s+(\d{4})")
|
||
RE_DMY = re.compile(r"(\d{2})\.(\d{2})\.(\d{4})")
|
||
|
||
|
||
def parse_ru_date(text: str | None) -> date | None:
|
||
"""Parse Russian date string like '9 мая 2026' → date(2026, 5, 9). Genitive month form only."""
|
||
if not text:
|
||
return None
|
||
m = RE_RU_DATE.search(text)
|
||
if not m:
|
||
return None
|
||
day_s, month_ru, year_s = m.groups()
|
||
month = RU_MONTHS.get(month_ru.lower())
|
||
if not month:
|
||
return None
|
||
try:
|
||
return date(int(year_s), month, int(day_s))
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
def parse_dmy(text: str | None) -> date | None:
|
||
"""Parse DD.MM.YYYY format (used by Yandex Valuation tool history) → date."""
|
||
if not text:
|
||
return None
|
||
m = RE_DMY.search(text)
|
||
if not m:
|
||
return None
|
||
try:
|
||
return date(int(m.group(3)), int(m.group(2)), int(m.group(1)))
|
||
except (ValueError, TypeError):
|
||
return None
|
||
|
||
|
||
# Relative date parsing (Yandex SERP shows "N дней назад" / "N часов назад")
|
||
_RE_REL_DATE = re.compile(
|
||
r"(?P<n>\d+)\s+(?P<unit>"
|
||
r"секунд[ауы]?|"
|
||
r"минут[ауы]?|"
|
||
r"час(?:а|ов)?|"
|
||
r"день|дн(?:я|ей?)|"
|
||
r"недел[июяь]+|"
|
||
r"месяц(?:а|ев)?|"
|
||
r"год(?:а|ов)?"
|
||
r")\s+назад",
|
||
flags=re.I,
|
||
)
|
||
_REL_UNIT_DAYS: dict[str, int] = {
|
||
"секунд": 0, "минут": 0, "час": 0,
|
||
"день": 1, "дн": 1,
|
||
"недел": 7,
|
||
"месяц": 30,
|
||
"год": 365,
|
||
}
|
||
|
||
|
||
def _parse_relative_date_yandex(s: str | None) -> date | None:
|
||
"""Parse 'N дней/недель/месяцев назад' → date. Returns None if not matched."""
|
||
if not s:
|
||
return None
|
||
m = _RE_REL_DATE.search(s)
|
||
if not m:
|
||
return None
|
||
n = int(m["n"])
|
||
unit_raw = m["unit"].lower()
|
||
for prefix, days in _REL_UNIT_DAYS.items():
|
||
if unit_raw.startswith(prefix):
|
||
return date.today() - timedelta(days=n * days)
|
||
return None
|
||
|
||
|
||
def parse_listing_date(text: str | None) -> date | None:
|
||
"""Parse listing date from Yandex card text.
|
||
|
||
Tries in order:
|
||
1. Absolute Russian date: '9 мая 2026' → date(2026, 5, 9)
|
||
2. Relative date: 'N дней назад' → date.today() - N days
|
||
Returns None if neither matches.
|
||
"""
|
||
result = parse_ru_date(text)
|
||
if result is not None:
|
||
return result
|
||
return _parse_relative_date_yandex(text)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Section 4 — House type / class NLP from description text
|
||
# ---------------------------------------------------------------------------
|
||
|
||
|
||
def parse_house_type(text: str | None) -> str | None:
|
||
"""Extract house material type from free-form description.
|
||
|
||
Returns: 'panel' | 'monolith_brick' | 'monolith' | 'brick' | 'block' | 'wood' | None.
|
||
Order matters: 'monolith_brick' is checked before 'monolith' and 'brick' alone.
|
||
"""
|
||
if not text:
|
||
return None
|
||
t = text.lower()
|
||
if "панельн" in t:
|
||
return "panel"
|
||
if "монолит" in t and "кирпич" in t:
|
||
return "monolith_brick"
|
||
if "монолит" in t:
|
||
return "monolith"
|
||
if "кирпичн" in t:
|
||
return "brick"
|
||
if "блочн" in t:
|
||
return "block"
|
||
if "деревянн" in t:
|
||
return "wood"
|
||
return None
|
||
|
||
|
||
def parse_house_class(text: str | None) -> str | None:
|
||
"""Extract ЖК class from description.
|
||
|
||
Returns: 'elite' | 'premium' | 'business' | 'comfort_plus' | 'comfort' | 'economy' | None.
|
||
"""
|
||
if not text:
|
||
return None
|
||
t = text.lower()
|
||
if re.search(r"класса?\s+элит|элитный\s+класс", t):
|
||
return "elite"
|
||
if re.search(r"класса?\s+премиум|премиум.{0,5}класс", t):
|
||
return "premium"
|
||
if re.search(r"класса?\s+бизнес", t):
|
||
return "business"
|
||
if re.search(r"класса?\s+комфорт\s*\+", t):
|
||
return "comfort_plus"
|
||
if re.search(r"класса?\s+комфорт(?!\s*\+)", t):
|
||
return "comfort"
|
||
if re.search(r"класса?\s+эконом", t):
|
||
return "economy"
|
||
return None
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Section 5 — Money parsing
|
||
# ---------------------------------------------------------------------------
|
||
|
||
RE_RUB_MLN = re.compile(r"([\d,.]+)\s*млн\s*₽", re.IGNORECASE)
|
||
RE_RUB_RAW = re.compile(r"(\d[\d\s]*\d|\d)")
|
||
|
||
|
||
def parse_rub(text: str | None) -> int | None:
|
||
"""Parse Russian price string → integer rubles.
|
||
|
||
Supports:
|
||
'4 399 000 ₽' → 4_399_000
|
||
'4,4 млн ₽' → 4_400_000
|
||
'4.4 млн ₽' → 4_400_000
|
||
'117 500' → 117_500 (raw digits without currency, e.g. ppm2)
|
||
None / 'без цены' / '' → None
|
||
"""
|
||
if not text:
|
||
return None
|
||
text = text.strip()
|
||
if not text:
|
||
return None
|
||
|
||
m = RE_RUB_MLN.search(text)
|
||
if m:
|
||
try:
|
||
return int(float(m.group(1).replace(",", ".").replace(" ", "")) * 1_000_000)
|
||
except (ValueError, TypeError):
|
||
pass
|
||
|
||
m = RE_RUB_RAW.search(text)
|
||
if m:
|
||
cleaned = m.group(1).replace(" ", "")
|
||
try:
|
||
return int(cleaned)
|
||
except ValueError:
|
||
return None
|
||
return None
|