diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py b/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py new file mode 100644 index 00000000..8c1e0b77 --- /dev/null +++ b/tradein-mvp/backend/app/services/scrapers/yandex_helpers.py @@ -0,0 +1,243 @@ +"""Yandex Realty parsing helpers — JSON-LD + regex + DOM utilities for SERP, detail, newbuilding, +and valuation parsers.""" + +from __future__ import annotations + +import json +import re +from datetime import date +from typing import Any + +from selectolax.parser import HTMLParser + +__all__ = [ + "RE_AGENCY_FOUNDED", + "RE_AGENCY_OBJECTS", + "RE_FLOOR", + "RE_JK_ID", + "RE_METRO_WALK", + "RE_OFFER_ID", + "RE_PPM2", + "RE_PRICE", + "RE_STREET_ID", + "RE_TITLE_AREA", + "RE_TITLE_ROOMS", + "RE_VIEWS", + "RE_YEAR", + "RU_MONTHS", + "RU_MONTH_NOMINATIVE", + "extract_json_ld", + "find_ld_by_type", + "parse_dmy", + "parse_house_class", + "parse_house_type", + "parse_ru_date", + "parse_rub", +] + +# --------------------------------------------------------------------------- +# Section 1 — JSON-LD extraction +# --------------------------------------------------------------------------- + + +def extract_json_ld(html: str) -> list[dict[str, Any]]: + """Extract all " + ) + ld = extract_json_ld(html) + assert len(ld) == 1 + assert ld[0]["@type"] == "Product" + + +def test_extract_json_ld_multiple(): + html = ( + '' + '' + ) + ld = extract_json_ld(html) + assert len(ld) == 2 + + +def test_extract_json_ld_array_wrapper(): + html = '' + ld = extract_json_ld(html) + assert len(ld) == 2 + + +def test_extract_json_ld_invalid_skipped(): + html = ( + '' + '' + ) + ld = extract_json_ld(html) + assert len(ld) == 1 + assert ld[0]["@type"] == "Valid" + + +def test_find_ld_by_type_simple(): + html = ( + '' + '' + ) + product = find_ld_by_type(html, "Product") + assert product is not None + assert product["price"] == 100 + + +def test_find_ld_by_type_list_form(): + # @type can be a list + html = '' + assert find_ld_by_type(html, "Offer") is not None + + +def test_find_ld_by_type_missing(): + html = '' + assert find_ld_by_type(html, "Product") is None + + +# --- regex sanity --- +def test_offer_id(): + m = RE_OFFER_ID.search("https://realty.yandex.ru/offer/7567094292504417257/") + assert m and m.group(1) == "7567094292504417257" + + +def test_jk_id(): + m = RE_JK_ID.search("/ekaterinburg/kupit/novostrojka/tatlin-1592987/") + assert m + assert m.group(1) == "tatlin" + assert m.group(2) == "1592987" + + +def test_title_parsing(): + title = "39 м² · 1-комнатная квартира · 6 этаж из 15" + area = RE_TITLE_AREA.search(title) + rooms = RE_TITLE_ROOMS.search(title) + floor = RE_FLOOR.search(title) + assert area and area.group(1) == "39" + assert rooms and rooms.group(1) == "1" + assert floor and floor.group(1) == "6" and floor.group(2) == "15" + + +def test_title_studio(): + rooms = RE_TITLE_ROOMS.search("Студия 28 м²") + # group(2) captures студия/студию/студий when no number-room pattern present + assert rooms is not None + matched = rooms.group(1) or rooms.group(2) + assert matched is not None and matched.lower().startswith("студи")