feat(tradein): yandex_helpers.py — JSON-LD + regex + date + NLP + money helpers (#456)

tradein-mvp/backend/app/services/scrapers/yandex_helpers.py

@@ -0,0 +1,243 @@
+"""Yandex Realty parsing helpers — JSON-LD + regex + DOM utilities for SERP, detail, newbuilding,
+and valuation parsers."""
+
+from __future__ import annotations
+
+import json
+import re
+from datetime import date
+from typing import Any
+
+from selectolax.parser import HTMLParser
+
+__all__ = [
+    "RE_AGENCY_FOUNDED",
+    "RE_AGENCY_OBJECTS",
+    "RE_FLOOR",
+    "RE_JK_ID",
+    "RE_METRO_WALK",
+    "RE_OFFER_ID",
+    "RE_PPM2",
+    "RE_PRICE",
+    "RE_STREET_ID",
+    "RE_TITLE_AREA",
+    "RE_TITLE_ROOMS",
+    "RE_VIEWS",
+    "RE_YEAR",
+    "RU_MONTHS",
+    "RU_MONTH_NOMINATIVE",
+    "extract_json_ld",
+    "find_ld_by_type",
+    "parse_dmy",
+    "parse_house_class",
+    "parse_house_type",
+    "parse_ru_date",
+    "parse_rub",
+]
+
+# ---------------------------------------------------------------------------
+# Section 1 — JSON-LD extraction
+# ---------------------------------------------------------------------------
+
+
+def extract_json_ld(html: str) -> list[dict[str, Any]]:
+    """Extract all <script type='application/ld+json'> blocks from HTML.
+
+    Returns a list of parsed JSON-LD dicts. Skips blocks that fail to parse.
+    """
+    tree = HTMLParser(html)
+    results: list[dict[str, Any]] = []
+    for script in tree.css('script[type="application/ld+json"]'):
+        text = script.text() or ""
+        if not text.strip():
+            continue
+        try:
+            parsed = json.loads(text)
+        except json.JSONDecodeError:
+            continue
+        if isinstance(parsed, dict):
+            results.append(parsed)
+        elif isinstance(parsed, list):
+            # Some pages wrap multiple LDs in a JSON array
+            results.extend(item for item in parsed if isinstance(item, dict))
+    return results
+
+
+def find_ld_by_type(html: str, type_name: str) -> dict[str, Any] | None:
+    """Return the first JSON-LD dict whose @type equals type_name.
+
+    If @type is a list, returns the dict if type_name is contained in it.
+    """
+    for ld in extract_json_ld(html):
+        t = ld.get("@type")
+        if t == type_name:
+            return ld
+        if isinstance(t, list) and type_name in t:
+            return ld
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Section 2 — Regex extractors (module-level compiled)
+# ---------------------------------------------------------------------------
+
+# URL pattern extractors
+RE_OFFER_ID = re.compile(r"/offer/(\d+)/?")
+RE_JK_ID = re.compile(r"/novostrojka/([\w-]+?)-(\d+)/?")          # group(1)=slug, group(2)=id
+RE_STREET_ID = re.compile(r"/kupit/kvartira/st-([\w-]+?)-(\d+)/?")
+
+# Card / title text extractors
+RE_TITLE_AREA = re.compile(r"(\d+[.,]?\d*)\s*м²")
+RE_TITLE_ROOMS = re.compile(r"(\d+)\s*-?\s*комнатн|(студи[яюй])", re.IGNORECASE)
+RE_FLOOR = re.compile(r"(\d+)\s+этаж\s+из\s+(\d+)", re.IGNORECASE)
+RE_PRICE = re.compile(r"(\d[\d\s]+\d)\s*₽")
+RE_PPM2 = re.compile(r"(\d[\d\s]+)\s*₽\s*за\s*м²", re.IGNORECASE)
+RE_VIEWS = re.compile(r"(\d+)\s+просмотр", re.IGNORECASE)
+
+# Detail / NLP extractors
+RE_METRO_WALK = re.compile(
+    r"(\d+)\s+минут\s+(?:неспешной\s+прогулки|ходьбы|пешком)", re.IGNORECASE
+)
+RE_YEAR = re.compile(r"\b(19\d{2}|20[0-2]\d)\b")
+
+# Agency block ("Год основания 1995", "150 объектов")
+RE_AGENCY_FOUNDED = re.compile(r"Год\s+основания\s+(\d{4})", re.IGNORECASE)
+RE_AGENCY_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE)
+
+# ---------------------------------------------------------------------------
+# Section 3 — Russian date parsing
+# ---------------------------------------------------------------------------
+
+RU_MONTHS: dict[str, int] = {
+    "января": 1, "февраля": 2, "марта": 3, "апреля": 4, "мая": 5, "июня": 6,
+    "июля": 7, "августа": 8, "сентября": 9, "октября": 10, "ноября": 11, "декабря": 12,
+}
+RU_MONTH_NOMINATIVE: dict[str, int] = {
+    "январь": 1, "февраль": 2, "март": 3, "апрель": 4, "май": 5, "июнь": 6,
+    "июль": 7, "август": 8, "сентябрь": 9, "октябрь": 10, "ноябрь": 11, "декабрь": 12,
+}
+RE_RU_DATE = re.compile(r"(\d{1,2})\s+(\w+)\s+(\d{4})")
+RE_DMY = re.compile(r"(\d{2})\.(\d{2})\.(\d{4})")
+
+
+def parse_ru_date(text: str | None) -> date | None:
+    """Parse Russian date string like '9 мая 2026' → date(2026, 5, 9). Genitive month form only."""
+    if not text:
+        return None
+    m = RE_RU_DATE.search(text)
+    if not m:
+        return None
+    day_s, month_ru, year_s = m.groups()
+    month = RU_MONTHS.get(month_ru.lower())
+    if not month:
+        return None
+    try:
+        return date(int(year_s), month, int(day_s))
+    except (ValueError, TypeError):
+        return None
+
+
+def parse_dmy(text: str | None) -> date | None:
+    """Parse DD.MM.YYYY format (used by Yandex Valuation tool history) → date."""
+    if not text:
+        return None
+    m = RE_DMY.search(text)
+    if not m:
+        return None
+    try:
+        return date(int(m.group(3)), int(m.group(2)), int(m.group(1)))
+    except (ValueError, TypeError):
+        return None
+
+
+# ---------------------------------------------------------------------------
+# Section 4 — House type / class NLP from description text
+# ---------------------------------------------------------------------------
+
+
+def parse_house_type(text: str | None) -> str | None:
+    """Extract house material type from free-form description.
+
+    Returns: 'panel' | 'monolith_brick' | 'monolith' | 'brick' | 'block' | 'wood' | None.
+    Order matters: 'monolith_brick' is checked before 'monolith' and 'brick' alone.
+    """
+    if not text:
+        return None
+    t = text.lower()
+    if "панельн" in t:
+        return "panel"
+    if "монолит" in t and "кирпич" in t:
+        return "monolith_brick"
+    if "монолит" in t:
+        return "monolith"
+    if "кирпичн" in t:
+        return "brick"
+    if "блочн" in t:
+        return "block"
+    if "деревянн" in t:
+        return "wood"
+    return None
+
+
+def parse_house_class(text: str | None) -> str | None:
+    """Extract ЖК class from description.
+
+    Returns: 'elite' | 'premium' | 'business' | 'comfort_plus' | 'comfort' | 'economy' | None.
+    """
+    if not text:
+        return None
+    t = text.lower()
+    if re.search(r"класса?\s+элит|элитный\s+класс", t):
+        return "elite"
+    if re.search(r"класса?\s+премиум|премиум.{0,5}класс", t):
+        return "premium"
+    if re.search(r"класса?\s+бизнес", t):
+        return "business"
+    if re.search(r"класса?\s+комфорт\s*\+", t):
+        return "comfort_plus"
+    if re.search(r"класса?\s+комфорт(?!\s*\+)", t):
+        return "comfort"
+    if re.search(r"класса?\s+эконом", t):
+        return "economy"
+    return None
+
+
+# ---------------------------------------------------------------------------
+# Section 5 — Money parsing
+# ---------------------------------------------------------------------------
+
+RE_RUB_MLN = re.compile(r"([\d,.]+)\s*млн\s*₽", re.IGNORECASE)
+RE_RUB_RAW = re.compile(r"(\d[\d\s]*\d|\d)")
+
+
+def parse_rub(text: str | None) -> int | None:
+    """Parse Russian price string → integer rubles.
+
+    Supports:
+      '4 399 000 ₽'   → 4_399_000
+      '4,4 млн ₽'     → 4_400_000
+      '4.4 млн ₽'     → 4_400_000
+      '117 500'       → 117_500   (raw digits without currency, e.g. ppm2)
+      None / 'без цены' / ''      → None
+    """
+    if not text:
+        return None
+    text = text.strip()
+    if not text:
+        return None
+
+    m = RE_RUB_MLN.search(text)
+    if m:
+        try:
+            return int(float(m.group(1).replace(",", ".").replace(" ", "")) * 1_000_000)
+        except (ValueError, TypeError):
+            pass
+
+    m = RE_RUB_RAW.search(text)
+    if m:
+        cleaned = m.group(1).replace(" ", "")
+        try:
+            return int(cleaned)
+        except ValueError:
+            return None
+    return None

tradein-mvp/backend/tests/test_yandex_helpers.py

@@ -0,0 +1,220 @@
+"""Unit tests for Yandex Realty parsing helpers."""
+
+from datetime import date
+
+from app.services.scrapers.yandex_helpers import (
+    RE_FLOOR,
+    RE_JK_ID,
+    RE_OFFER_ID,
+    RE_TITLE_AREA,
+    RE_TITLE_ROOMS,
+    extract_json_ld,
+    find_ld_by_type,
+    parse_dmy,
+    parse_house_class,
+    parse_house_type,
+    parse_ru_date,
+    parse_rub,
+)
+
+
+# --- house_type ---
+def test_parse_house_type_panel():
+    assert parse_house_type("в тёплом, панельном доме 1981 года") == "panel"
+
+
+def test_parse_house_type_monolith_brick():
+    assert parse_house_type("кирпично-монолитный дом") == "monolith_brick"
+
+
+def test_parse_house_type_monolith_only():
+    assert parse_house_type("монолитное здание") == "monolith"
+
+
+def test_parse_house_type_brick_only():
+    assert parse_house_type("кирпичный дом 1995") == "brick"
+
+
+def test_parse_house_type_unknown():
+    assert parse_house_type("из железобетона") is None
+    assert parse_house_type("") is None
+    assert parse_house_type(None) is None
+
+
+# --- house_class ---
+def test_parse_house_class_comfort_plus():
+    assert parse_house_class("дома класса комфорт+") == "comfort_plus"
+
+
+def test_parse_house_class_comfort():
+    assert parse_house_class("ЖК класса комфорт в центре") == "comfort"
+
+
+def test_parse_house_class_business():
+    assert parse_house_class("проект класса бизнес") == "business"
+
+
+def test_parse_house_class_premium():
+    assert parse_house_class("ЖК премиум-класса") == "premium"
+
+
+def test_parse_house_class_elite():
+    assert parse_house_class("элитный класс жилья") == "elite"
+
+
+def test_parse_house_class_none():
+    assert parse_house_class("обычный жилой комплекс") is None
+    assert parse_house_class(None) is None
+
+
+# --- parse_rub ---
+def test_parse_rub_full_price():
+    assert parse_rub("4 399 000 ₽") == 4_399_000
+
+
+def test_parse_rub_mln_comma():
+    assert parse_rub("4,4 млн ₽") == 4_400_000
+
+
+def test_parse_rub_mln_dot():
+    assert parse_rub("4.4 млн ₽") == 4_400_000
+
+
+def test_parse_rub_ppm2():
+    assert parse_rub("117 500 ₽ за м²") == 117_500
+
+
+def test_parse_rub_raw_digits():
+    assert parse_rub("117500") == 117_500
+
+
+def test_parse_rub_empty():
+    assert parse_rub("") is None
+    assert parse_rub(None) is None
+    assert parse_rub("без цены") is None
+
+
+# --- parse_ru_date ---
+def test_parse_ru_date_basic():
+    assert parse_ru_date("9 мая 2026") == date(2026, 5, 9)
+
+
+def test_parse_ru_date_in_sentence():
+    assert parse_ru_date("опубликовано 17 февраля 2024") == date(2024, 2, 17)
+
+
+def test_parse_ru_date_invalid_month():
+    assert parse_ru_date("9 invalidmonth 2026") is None
+
+
+def test_parse_ru_date_dmy_not_supported():
+    # parse_ru_date doesn't handle DD.MM.YYYY (that's parse_dmy's job)
+    assert parse_ru_date("01.02.2026") is None
+
+
+def test_parse_ru_date_empty():
+    assert parse_ru_date("") is None
+    assert parse_ru_date(None) is None
+
+
+# --- parse_dmy ---
+def test_parse_dmy_basic():
+    assert parse_dmy("17.02.2026") == date(2026, 2, 17)
+
+
+def test_parse_dmy_in_sentence():
+    assert parse_dmy("Дата 31.12.2025 — конец года") == date(2025, 12, 31)
+
+
+def test_parse_dmy_invalid():
+    assert parse_dmy("99.99.9999") is None
+    assert parse_dmy("") is None
+    assert parse_dmy(None) is None
+
+
+# --- JSON-LD ---
+def test_extract_json_ld_single():
+    html = (
+        '<html><script type="application/ld+json">{"@type": "Product", "name": "test"}'
+        "</script></html>"
+    )
+    ld = extract_json_ld(html)
+    assert len(ld) == 1
+    assert ld[0]["@type"] == "Product"
+
+
+def test_extract_json_ld_multiple():
+    html = (
+        '<script type="application/ld+json">{"@type": "Product"}</script>'
+        '<script type="application/ld+json">{"@type": "Organization"}</script>'
+    )
+    ld = extract_json_ld(html)
+    assert len(ld) == 2
+
+
+def test_extract_json_ld_array_wrapper():
+    html = '<script type="application/ld+json">[{"@type": "A"}, {"@type": "B"}]</script>'
+    ld = extract_json_ld(html)
+    assert len(ld) == 2
+
+
+def test_extract_json_ld_invalid_skipped():
+    html = (
+        '<script type="application/ld+json">not valid json</script>'
+        '<script type="application/ld+json">{"@type": "Valid"}</script>'
+    )
+    ld = extract_json_ld(html)
+    assert len(ld) == 1
+    assert ld[0]["@type"] == "Valid"
+
+
+def test_find_ld_by_type_simple():
+    html = (
+        '<script type="application/ld+json">{"@type": "Product", "price": 100}</script>'
+        '<script type="application/ld+json">{"@type": "BreadcrumbList"}</script>'
+    )
+    product = find_ld_by_type(html, "Product")
+    assert product is not None
+    assert product["price"] == 100
+
+
+def test_find_ld_by_type_list_form():
+    # @type can be a list
+    html = '<script type="application/ld+json">{"@type": ["Product", "Offer"], "x": 1}</script>'
+    assert find_ld_by_type(html, "Offer") is not None
+
+
+def test_find_ld_by_type_missing():
+    html = '<script type="application/ld+json">{"@type": "Other"}</script>'
+    assert find_ld_by_type(html, "Product") is None
+
+
+# --- regex sanity ---
+def test_offer_id():
+    m = RE_OFFER_ID.search("https://realty.yandex.ru/offer/7567094292504417257/")
+    assert m and m.group(1) == "7567094292504417257"
+
+
+def test_jk_id():
+    m = RE_JK_ID.search("/ekaterinburg/kupit/novostrojka/tatlin-1592987/")
+    assert m
+    assert m.group(1) == "tatlin"
+    assert m.group(2) == "1592987"
+
+
+def test_title_parsing():
+    title = "39 м² · 1-комнатная квартира · 6 этаж из 15"
+    area = RE_TITLE_AREA.search(title)
+    rooms = RE_TITLE_ROOMS.search(title)
+    floor = RE_FLOOR.search(title)
+    assert area and area.group(1) == "39"
+    assert rooms and rooms.group(1) == "1"
+    assert floor and floor.group(1) == "6" and floor.group(2) == "15"
+
+
+def test_title_studio():
+    rooms = RE_TITLE_ROOMS.search("Студия 28 м²")
+    # group(2) captures студия/студию/студий when no number-room pattern present
+    assert rooms is not None
+    matched = rooms.group(1) or rooms.group(2)
+    assert matched is not None and matched.lower().startswith("студи")