220 lines
6.2 KiB
Python
220 lines
6.2 KiB
Python
"""Unit tests for Yandex Realty parsing helpers."""
|
|
|
|
from datetime import date
|
|
|
|
from app.services.scrapers.yandex_helpers import (
|
|
RE_FLOOR,
|
|
RE_JK_ID,
|
|
RE_OFFER_ID,
|
|
RE_TITLE_AREA,
|
|
RE_TITLE_ROOMS,
|
|
extract_json_ld,
|
|
find_ld_by_type,
|
|
parse_dmy,
|
|
parse_house_class,
|
|
parse_house_type,
|
|
parse_ru_date,
|
|
parse_rub,
|
|
)
|
|
|
|
|
|
# --- house_type ---
|
|
def test_parse_house_type_panel():
|
|
assert parse_house_type("в тёплом, панельном доме 1981 года") == "panel"
|
|
|
|
|
|
def test_parse_house_type_monolith_brick():
|
|
assert parse_house_type("кирпично-монолитный дом") == "monolith_brick"
|
|
|
|
|
|
def test_parse_house_type_monolith_only():
|
|
assert parse_house_type("монолитное здание") == "monolith"
|
|
|
|
|
|
def test_parse_house_type_brick_only():
|
|
assert parse_house_type("кирпичный дом 1995") == "brick"
|
|
|
|
|
|
def test_parse_house_type_unknown():
|
|
assert parse_house_type("из железобетона") is None
|
|
assert parse_house_type("") is None
|
|
assert parse_house_type(None) is None
|
|
|
|
|
|
# --- house_class ---
|
|
def test_parse_house_class_comfort_plus():
|
|
assert parse_house_class("дома класса комфорт+") == "comfort_plus"
|
|
|
|
|
|
def test_parse_house_class_comfort():
|
|
assert parse_house_class("ЖК класса комфорт в центре") == "comfort"
|
|
|
|
|
|
def test_parse_house_class_business():
|
|
assert parse_house_class("проект класса бизнес") == "business"
|
|
|
|
|
|
def test_parse_house_class_premium():
|
|
assert parse_house_class("ЖК премиум-класса") == "premium"
|
|
|
|
|
|
def test_parse_house_class_elite():
|
|
assert parse_house_class("элитный класс жилья") == "elite"
|
|
|
|
|
|
def test_parse_house_class_none():
|
|
assert parse_house_class("обычный жилой комплекс") is None
|
|
assert parse_house_class(None) is None
|
|
|
|
|
|
# --- parse_rub ---
|
|
def test_parse_rub_full_price():
|
|
assert parse_rub("4 399 000 ₽") == 4_399_000
|
|
|
|
|
|
def test_parse_rub_mln_comma():
|
|
assert parse_rub("4,4 млн ₽") == 4_400_000
|
|
|
|
|
|
def test_parse_rub_mln_dot():
|
|
assert parse_rub("4.4 млн ₽") == 4_400_000
|
|
|
|
|
|
def test_parse_rub_ppm2():
|
|
assert parse_rub("117 500 ₽ за м²") == 117_500
|
|
|
|
|
|
def test_parse_rub_raw_digits():
|
|
assert parse_rub("117500") == 117_500
|
|
|
|
|
|
def test_parse_rub_empty():
|
|
assert parse_rub("") is None
|
|
assert parse_rub(None) is None
|
|
assert parse_rub("без цены") is None
|
|
|
|
|
|
# --- parse_ru_date ---
|
|
def test_parse_ru_date_basic():
|
|
assert parse_ru_date("9 мая 2026") == date(2026, 5, 9)
|
|
|
|
|
|
def test_parse_ru_date_in_sentence():
|
|
assert parse_ru_date("опубликовано 17 февраля 2024") == date(2024, 2, 17)
|
|
|
|
|
|
def test_parse_ru_date_invalid_month():
|
|
assert parse_ru_date("9 invalidmonth 2026") is None
|
|
|
|
|
|
def test_parse_ru_date_dmy_not_supported():
|
|
# parse_ru_date doesn't handle DD.MM.YYYY (that's parse_dmy's job)
|
|
assert parse_ru_date("01.02.2026") is None
|
|
|
|
|
|
def test_parse_ru_date_empty():
|
|
assert parse_ru_date("") is None
|
|
assert parse_ru_date(None) is None
|
|
|
|
|
|
# --- parse_dmy ---
|
|
def test_parse_dmy_basic():
|
|
assert parse_dmy("17.02.2026") == date(2026, 2, 17)
|
|
|
|
|
|
def test_parse_dmy_in_sentence():
|
|
assert parse_dmy("Дата 31.12.2025 — конец года") == date(2025, 12, 31)
|
|
|
|
|
|
def test_parse_dmy_invalid():
|
|
assert parse_dmy("99.99.9999") is None
|
|
assert parse_dmy("") is None
|
|
assert parse_dmy(None) is None
|
|
|
|
|
|
# --- JSON-LD ---
|
|
def test_extract_json_ld_single():
|
|
html = (
|
|
'<html><script type="application/ld+json">{"@type": "Product", "name": "test"}'
|
|
"</script></html>"
|
|
)
|
|
ld = extract_json_ld(html)
|
|
assert len(ld) == 1
|
|
assert ld[0]["@type"] == "Product"
|
|
|
|
|
|
def test_extract_json_ld_multiple():
|
|
html = (
|
|
'<script type="application/ld+json">{"@type": "Product"}</script>'
|
|
'<script type="application/ld+json">{"@type": "Organization"}</script>'
|
|
)
|
|
ld = extract_json_ld(html)
|
|
assert len(ld) == 2
|
|
|
|
|
|
def test_extract_json_ld_array_wrapper():
|
|
html = '<script type="application/ld+json">[{"@type": "A"}, {"@type": "B"}]</script>'
|
|
ld = extract_json_ld(html)
|
|
assert len(ld) == 2
|
|
|
|
|
|
def test_extract_json_ld_invalid_skipped():
|
|
html = (
|
|
'<script type="application/ld+json">not valid json</script>'
|
|
'<script type="application/ld+json">{"@type": "Valid"}</script>'
|
|
)
|
|
ld = extract_json_ld(html)
|
|
assert len(ld) == 1
|
|
assert ld[0]["@type"] == "Valid"
|
|
|
|
|
|
def test_find_ld_by_type_simple():
|
|
html = (
|
|
'<script type="application/ld+json">{"@type": "Product", "price": 100}</script>'
|
|
'<script type="application/ld+json">{"@type": "BreadcrumbList"}</script>'
|
|
)
|
|
product = find_ld_by_type(html, "Product")
|
|
assert product is not None
|
|
assert product["price"] == 100
|
|
|
|
|
|
def test_find_ld_by_type_list_form():
|
|
# @type can be a list
|
|
html = '<script type="application/ld+json">{"@type": ["Product", "Offer"], "x": 1}</script>'
|
|
assert find_ld_by_type(html, "Offer") is not None
|
|
|
|
|
|
def test_find_ld_by_type_missing():
|
|
html = '<script type="application/ld+json">{"@type": "Other"}</script>'
|
|
assert find_ld_by_type(html, "Product") is None
|
|
|
|
|
|
# --- regex sanity ---
|
|
def test_offer_id():
|
|
m = RE_OFFER_ID.search("https://realty.yandex.ru/offer/7567094292504417257/")
|
|
assert m and m.group(1) == "7567094292504417257"
|
|
|
|
|
|
def test_jk_id():
|
|
m = RE_JK_ID.search("/ekaterinburg/kupit/novostrojka/tatlin-1592987/")
|
|
assert m
|
|
assert m.group(1) == "tatlin"
|
|
assert m.group(2) == "1592987"
|
|
|
|
|
|
def test_title_parsing():
|
|
title = "39 м² · 1-комнатная квартира · 6 этаж из 15"
|
|
area = RE_TITLE_AREA.search(title)
|
|
rooms = RE_TITLE_ROOMS.search(title)
|
|
floor = RE_FLOOR.search(title)
|
|
assert area and area.group(1) == "39"
|
|
assert rooms and rooms.group(1) == "1"
|
|
assert floor and floor.group(1) == "6" and floor.group(2) == "15"
|
|
|
|
|
|
def test_title_studio():
|
|
rooms = RE_TITLE_ROOMS.search("Студия 28 м²")
|
|
# group(2) captures студия/студию/студий when no number-room pattern present
|
|
assert rooms is not None
|
|
matched = rooms.group(1) or rooms.group(2)
|
|
assert matched is not None and matched.lower().startswith("студи")
|