Light1YT
bdb9f3dd09
Production: Yandex Realty имеет 60% listings с listing_date=NULL (2234/3704).
Audit показал parse_listing_date() ловил только absolute с годом
(«9 мая 2026») и relative numeric («5 дней назад»), но Yandex SERP
для свежих cards чаще использует:
- Keyword: «сегодня в 12:30», «вчера», «позавчера», «только что»
- Absolute без года: «18 мая» (год inferred)
Changes:
- yandex_helpers.py:
* RE_RU_DATE_NO_YEAR — regex для absolute без \d{4}
* _RU_MONTH_PREFIXES dict для day+month → date
* parse_listing_date(): 4 fallback tiers
1) keyword shortcuts
2) absolute с годом (existing)
3) absolute без года → infer current year, rollback на prev если future >30d
4) relative numeric (existing)
- tests: 3 new test groups (keyword/absolute-no-year/relative-still-works),
41/41 pass
Impact: linkage rate в /sales-vs-listings (PR K) для Yandex pairs
повысится с 40% → ~90% после next sweep. Existing 2234 NULL — будут
re-extracted при periodic scrape (URL остался).
Mirrors PR N (Avito same fix). Pattern теперь consistent между scrapers.
Refs #564.
271 lines
8.4 KiB
Python
271 lines
8.4 KiB
Python
"""Unit tests for Yandex Realty parsing helpers."""
|
||
|
||
from datetime import date
|
||
|
||
from app.services.scrapers.yandex_helpers import (
|
||
RE_FLOOR,
|
||
RE_JK_ID,
|
||
RE_OFFER_ID,
|
||
RE_TITLE_AREA,
|
||
RE_TITLE_ROOMS,
|
||
extract_json_ld,
|
||
find_ld_by_type,
|
||
parse_dmy,
|
||
parse_house_class,
|
||
parse_house_type,
|
||
parse_ru_date,
|
||
parse_rub,
|
||
)
|
||
|
||
|
||
# --- house_type ---
|
||
def test_parse_house_type_panel():
|
||
assert parse_house_type("в тёплом, панельном доме 1981 года") == "panel"
|
||
|
||
|
||
def test_parse_house_type_monolith_brick():
|
||
assert parse_house_type("кирпично-монолитный дом") == "monolith_brick"
|
||
|
||
|
||
def test_parse_house_type_monolith_only():
|
||
assert parse_house_type("монолитное здание") == "monolith"
|
||
|
||
|
||
def test_parse_house_type_brick_only():
|
||
assert parse_house_type("кирпичный дом 1995") == "brick"
|
||
|
||
|
||
def test_parse_house_type_unknown():
|
||
assert parse_house_type("из железобетона") is None
|
||
assert parse_house_type("") is None
|
||
assert parse_house_type(None) is None
|
||
|
||
|
||
# --- house_class ---
|
||
def test_parse_house_class_comfort_plus():
|
||
assert parse_house_class("дома класса комфорт+") == "comfort_plus"
|
||
|
||
|
||
def test_parse_house_class_comfort():
|
||
assert parse_house_class("ЖК класса комфорт в центре") == "comfort"
|
||
|
||
|
||
def test_parse_house_class_business():
|
||
assert parse_house_class("проект класса бизнес") == "business"
|
||
|
||
|
||
def test_parse_house_class_premium():
|
||
assert parse_house_class("ЖК премиум-класса") == "premium"
|
||
|
||
|
||
def test_parse_house_class_elite():
|
||
assert parse_house_class("элитный класс жилья") == "elite"
|
||
|
||
|
||
def test_parse_house_class_none():
|
||
assert parse_house_class("обычный жилой комплекс") is None
|
||
assert parse_house_class(None) is None
|
||
|
||
|
||
# --- parse_rub ---
|
||
def test_parse_rub_full_price():
|
||
assert parse_rub("4 399 000 ₽") == 4_399_000
|
||
|
||
|
||
def test_parse_rub_mln_comma():
|
||
assert parse_rub("4,4 млн ₽") == 4_400_000
|
||
|
||
|
||
def test_parse_rub_mln_dot():
|
||
assert parse_rub("4.4 млн ₽") == 4_400_000
|
||
|
||
|
||
def test_parse_rub_ppm2():
|
||
assert parse_rub("117 500 ₽ за м²") == 117_500
|
||
|
||
|
||
def test_parse_rub_raw_digits():
|
||
assert parse_rub("117500") == 117_500
|
||
|
||
|
||
def test_parse_rub_empty():
|
||
assert parse_rub("") is None
|
||
assert parse_rub(None) is None
|
||
assert parse_rub("без цены") is None
|
||
|
||
|
||
def test_parse_rub_nbsp_separator():
|
||
"""Yandex SERP форматирует «9\\xa0800\\xa0000» через NBSP, не regular space.
|
||
|
||
До fix'а `.replace(" ", "")` оставлял \\xa0 → int() падал ValueError →
|
||
parse_rub() возвращал None → весь scraper не парсил lots (0/23 на live HTML).
|
||
"""
|
||
assert parse_rub("9\xa0800\xa0000\xa0₽") == 9_800_000
|
||
assert parse_rub("4\xa0399\xa0000") == 4_399_000
|
||
|
||
|
||
def test_parse_rub_mln_with_nbsp():
|
||
"""«4,4\\xa0млн\\xa0₽» — NBSP вокруг unit-keyword тоже встречается."""
|
||
assert parse_rub("4,4\xa0млн\xa0₽") == 4_400_000
|
||
|
||
|
||
# --- parse_ru_date ---
|
||
def test_parse_ru_date_basic():
|
||
assert parse_ru_date("9 мая 2026") == date(2026, 5, 9)
|
||
|
||
|
||
def test_parse_ru_date_in_sentence():
|
||
assert parse_ru_date("опубликовано 17 февраля 2024") == date(2024, 2, 17)
|
||
|
||
|
||
def test_parse_ru_date_invalid_month():
|
||
assert parse_ru_date("9 invalidmonth 2026") is None
|
||
|
||
|
||
def test_parse_ru_date_dmy_not_supported():
|
||
# parse_ru_date doesn't handle DD.MM.YYYY (that's parse_dmy's job)
|
||
assert parse_ru_date("01.02.2026") is None
|
||
|
||
|
||
def test_parse_ru_date_empty():
|
||
assert parse_ru_date("") is None
|
||
assert parse_ru_date(None) is None
|
||
|
||
|
||
# --- parse_listing_date (extended formats) ---
|
||
def test_listing_date_today_keyword():
|
||
from datetime import timedelta
|
||
|
||
from app.services.scrapers.yandex_helpers import parse_listing_date
|
||
assert parse_listing_date("сегодня в 12:30") == date.today()
|
||
assert parse_listing_date("вчера в 18:45") == date.today() - timedelta(days=1)
|
||
assert parse_listing_date("позавчера") == date.today() - timedelta(days=2)
|
||
assert parse_listing_date("только что") == date.today()
|
||
|
||
|
||
def test_listing_date_absolute_no_year():
|
||
"""Yandex SERP формат «18 мая» без года — infer current year."""
|
||
from app.services.scrapers.yandex_helpers import parse_listing_date
|
||
today = date.today()
|
||
# Pick a date в прошлом этого года, чтобы тест стабильный
|
||
if today.month >= 2:
|
||
target = date(today.year, today.month - 1, 15)
|
||
else:
|
||
target = date(today.year - 1, 12, 15)
|
||
ru_months = {
|
||
1: "января", 2: "февраля", 3: "марта", 4: "апреля",
|
||
5: "мая", 6: "июня", 7: "июля", 8: "августа",
|
||
9: "сентября", 10: "октября", 11: "ноября", 12: "декабря",
|
||
}
|
||
text = f"{target.day} {ru_months[target.month]}"
|
||
assert parse_listing_date(text) == target
|
||
|
||
|
||
def test_listing_date_relative_still_works():
|
||
from datetime import timedelta
|
||
|
||
from app.services.scrapers.yandex_helpers import parse_listing_date
|
||
assert parse_listing_date("5 дней назад") == date.today() - timedelta(days=5)
|
||
|
||
|
||
# --- parse_dmy ---
|
||
def test_parse_dmy_basic():
|
||
assert parse_dmy("17.02.2026") == date(2026, 2, 17)
|
||
|
||
|
||
def test_parse_dmy_in_sentence():
|
||
assert parse_dmy("Дата 31.12.2025 — конец года") == date(2025, 12, 31)
|
||
|
||
|
||
def test_parse_dmy_invalid():
|
||
assert parse_dmy("99.99.9999") is None
|
||
assert parse_dmy("") is None
|
||
assert parse_dmy(None) is None
|
||
|
||
|
||
# --- JSON-LD ---
|
||
def test_extract_json_ld_single():
|
||
html = (
|
||
'<html><script type="application/ld+json">{"@type": "Product", "name": "test"}'
|
||
"</script></html>"
|
||
)
|
||
ld = extract_json_ld(html)
|
||
assert len(ld) == 1
|
||
assert ld[0]["@type"] == "Product"
|
||
|
||
|
||
def test_extract_json_ld_multiple():
|
||
html = (
|
||
'<script type="application/ld+json">{"@type": "Product"}</script>'
|
||
'<script type="application/ld+json">{"@type": "Organization"}</script>'
|
||
)
|
||
ld = extract_json_ld(html)
|
||
assert len(ld) == 2
|
||
|
||
|
||
def test_extract_json_ld_array_wrapper():
|
||
html = '<script type="application/ld+json">[{"@type": "A"}, {"@type": "B"}]</script>'
|
||
ld = extract_json_ld(html)
|
||
assert len(ld) == 2
|
||
|
||
|
||
def test_extract_json_ld_invalid_skipped():
|
||
html = (
|
||
'<script type="application/ld+json">not valid json</script>'
|
||
'<script type="application/ld+json">{"@type": "Valid"}</script>'
|
||
)
|
||
ld = extract_json_ld(html)
|
||
assert len(ld) == 1
|
||
assert ld[0]["@type"] == "Valid"
|
||
|
||
|
||
def test_find_ld_by_type_simple():
|
||
html = (
|
||
'<script type="application/ld+json">{"@type": "Product", "price": 100}</script>'
|
||
'<script type="application/ld+json">{"@type": "BreadcrumbList"}</script>'
|
||
)
|
||
product = find_ld_by_type(html, "Product")
|
||
assert product is not None
|
||
assert product["price"] == 100
|
||
|
||
|
||
def test_find_ld_by_type_list_form():
|
||
# @type can be a list
|
||
html = '<script type="application/ld+json">{"@type": ["Product", "Offer"], "x": 1}</script>'
|
||
assert find_ld_by_type(html, "Offer") is not None
|
||
|
||
|
||
def test_find_ld_by_type_missing():
|
||
html = '<script type="application/ld+json">{"@type": "Other"}</script>'
|
||
assert find_ld_by_type(html, "Product") is None
|
||
|
||
|
||
# --- regex sanity ---
|
||
def test_offer_id():
|
||
m = RE_OFFER_ID.search("https://realty.yandex.ru/offer/7567094292504417257/")
|
||
assert m and m.group(1) == "7567094292504417257"
|
||
|
||
|
||
def test_jk_id():
|
||
m = RE_JK_ID.search("/ekaterinburg/kupit/novostrojka/tatlin-1592987/")
|
||
assert m
|
||
assert m.group(1) == "tatlin"
|
||
assert m.group(2) == "1592987"
|
||
|
||
|
||
def test_title_parsing():
|
||
title = "39 м² · 1-комнатная квартира · 6 этаж из 15"
|
||
area = RE_TITLE_AREA.search(title)
|
||
rooms = RE_TITLE_ROOMS.search(title)
|
||
floor = RE_FLOOR.search(title)
|
||
assert area and area.group(1) == "39"
|
||
assert rooms and rooms.group(1) == "1"
|
||
assert floor and floor.group(1) == "6" and floor.group(2) == "15"
|
||
|
||
|
||
def test_title_studio():
|
||
rooms = RE_TITLE_ROOMS.search("Студия 28 м²")
|
||
# group(2) captures студия/студию/студий when no number-room pattern present
|
||
assert rooms is not None
|
||
matched = rooms.group(1) or rooms.group(2)
|
||
assert matched is not None and matched.lower().startswith("студи")
|