fix(yandex-valuation): parse removed_date, fix total_floors regex, normalize NBSP #526
2 changed files with 156 additions and 31 deletions
|
|
@ -52,6 +52,35 @@ class ValuationHouseMeta(BaseModel):
|
|||
total_objects: int | None = None # 'N объектов' (full archive count)
|
||||
has_panorama: bool = False # 'Панорама' label present
|
||||
|
||||
def validate_match(
|
||||
self,
|
||||
expected_year_built: int | None = None,
|
||||
expected_total_floors: int | None = None,
|
||||
) -> float:
|
||||
"""Return confidence 0..1 that this house meta matches expected values.
|
||||
|
||||
Used after fetching valuation by address to detect when Yandex returned a
|
||||
different house (ambiguous address geocoding). Tolerance ±1 year, ±1 floor.
|
||||
|
||||
Both expected=None → 1.0 (no check). Mismatch on any dimension → 0.0 for it.
|
||||
"""
|
||||
score = 0.0
|
||||
checks = 0
|
||||
if expected_year_built is not None:
|
||||
checks += 1
|
||||
if self.year_built is not None and abs(self.year_built - expected_year_built) <= 1:
|
||||
score += 1.0
|
||||
if expected_total_floors is not None:
|
||||
checks += 1
|
||||
if (
|
||||
self.total_floors is not None
|
||||
and abs(self.total_floors - expected_total_floors) <= 1
|
||||
):
|
||||
score += 1.0
|
||||
if checks == 0:
|
||||
return 1.0
|
||||
return score / checks
|
||||
|
||||
|
||||
class ValuationHistoryItem(BaseModel):
|
||||
"""One historical offer entry from the valuation page."""
|
||||
|
|
@ -64,6 +93,7 @@ class ValuationHistoryItem(BaseModel):
|
|||
last_price: int | None = None
|
||||
last_price_per_m2: int | None = None
|
||||
publish_date: date | None = None
|
||||
removed_date: date | None = None # ← NEW
|
||||
exposure_days: int | None = None
|
||||
status: str | None = None # 'В продаже' / 'Снято'
|
||||
|
||||
|
|
@ -86,7 +116,7 @@ class YandexValuationResult(BaseModel):
|
|||
# ---------------------------------------------------------------------------
|
||||
|
||||
RE_YEAR_BUILT = re.compile(r"Дом\s+(\d{4})\s+года", re.IGNORECASE)
|
||||
RE_FLOORS = re.compile(r"(\d+)\s+этаж", re.IGNORECASE)
|
||||
RE_FLOORS = re.compile(r"(\d+)\s+этажей", re.IGNORECASE)
|
||||
RE_CEILING = re.compile(r"([\d,]+)\s*м\s+потолки", re.IGNORECASE)
|
||||
RE_TOTAL_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE)
|
||||
|
||||
|
|
@ -98,9 +128,7 @@ RE_ITEM_EXPOSURE = re.compile(r"экспозиции\s+(\d+)\s+дн", re.IGNOREC
|
|||
RE_ITEM_STATUS = re.compile(r"(В\s+продаже|Снят[оа])", re.IGNORECASE)
|
||||
|
||||
# Matches total-price tokens (rubles) — excludes per-m2 tokens by negative lookahead
|
||||
_RE_PRICE_TOKEN = re.compile(
|
||||
r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)"
|
||||
)
|
||||
_RE_PRICE_TOKEN = re.compile(r"(?:\d[\d\s]*\d|\d)(?:[.,]\d+)?\s*(?:млн)?\s*₽(?!\s*за\s*м²)")
|
||||
_RE_PPM2_TOKEN = re.compile(r"\d[\d\s]*\s*₽\s*за\s*м²")
|
||||
|
||||
|
||||
|
|
@ -126,9 +154,7 @@ class YandexValuationScraper(BaseScraper):
|
|||
super().__init__()
|
||||
self.request_delay_sec = get_scraper_delay(self.name)
|
||||
|
||||
async def fetch_around(
|
||||
self, lat: float, lon: float, radius_m: int = 1000
|
||||
) -> list: # type: ignore[override]
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
"YandexValuationScraper is address-based; use fetch_house_history() instead."
|
||||
)
|
||||
|
|
@ -164,9 +190,7 @@ class YandexValuationScraper(BaseScraper):
|
|||
logger.exception("yandex valuation fetch failed: %s", url)
|
||||
return None
|
||||
if response.status_code != 200:
|
||||
logger.warning(
|
||||
"yandex valuation returned %d for %s", response.status_code, url
|
||||
)
|
||||
logger.warning("yandex valuation returned %d for %s", response.status_code, url)
|
||||
return None
|
||||
result = self.parse(
|
||||
response.text,
|
||||
|
|
@ -189,9 +213,10 @@ class YandexValuationScraper(BaseScraper):
|
|||
source_url: str,
|
||||
) -> YandexValuationResult:
|
||||
"""Parse raw HTML into YandexValuationResult. Pure function — usable in unit tests."""
|
||||
tree = HTMLParser(html)
|
||||
html_normalized = html.replace("\xa0", " ")
|
||||
tree = HTMLParser(html_normalized)
|
||||
body = tree.body
|
||||
body_text = body.text(strip=True) if body else ""
|
||||
body_text = (body.text(strip=True) if body else "").replace("\xa0", " ")
|
||||
|
||||
house = self._parse_house_meta(body_text)
|
||||
history_items = self._parse_history_items(tree, body_text)
|
||||
|
|
@ -221,17 +246,13 @@ class YandexValuationScraper(BaseScraper):
|
|||
year_built=int(year_m.group(1)) if year_m else None,
|
||||
total_floors=int(floors_m.group(1)) if floors_m else None,
|
||||
house_type=parse_house_type(body_text),
|
||||
ceiling_height=(
|
||||
float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None
|
||||
),
|
||||
ceiling_height=(float(ceiling_m.group(1).replace(",", ".")) if ceiling_m else None),
|
||||
has_lift="Лифт" in body_text,
|
||||
total_objects=int(objects_m.group(1)) if objects_m else None,
|
||||
has_panorama="Панорама" in body_text,
|
||||
)
|
||||
|
||||
def _parse_history_items(
|
||||
self, tree: HTMLParser, body_text: str
|
||||
) -> list[ValuationHistoryItem]:
|
||||
def _parse_history_items(self, tree: HTMLParser, body_text: str) -> list[ValuationHistoryItem]:
|
||||
"""Extract list of historical offer items using best available strategy.
|
||||
|
||||
Strategy 1: CSS data-test containers (if Yandex exposes them).
|
||||
|
|
@ -239,9 +260,7 @@ class YandexValuationScraper(BaseScraper):
|
|||
"""
|
||||
items: list[ValuationHistoryItem] = []
|
||||
# Strategy 1: explicit data-test container
|
||||
for container in tree.css(
|
||||
'[data-test*="HistoryItem"], [data-test*="ValuationItem"]'
|
||||
):
|
||||
for container in tree.css('[data-test*="HistoryItem"], [data-test*="ValuationItem"]'):
|
||||
item = self._parse_item_text(container.text(strip=True))
|
||||
if item:
|
||||
items.append(item)
|
||||
|
|
@ -284,7 +303,15 @@ class YandexValuationScraper(BaseScraper):
|
|||
start_ppm2 = parse_rub(ppm2_tokens[0]) if len(ppm2_tokens) >= 1 else None
|
||||
last_ppm2 = parse_rub(ppm2_tokens[1]) if len(ppm2_tokens) >= 2 else None
|
||||
|
||||
publish_date = parse_dmy(text)
|
||||
# Extract ALL DD.MM.YYYY dates: first → publish_date, second → removed_date
|
||||
date_matches = list(re.finditer(r"\d{2}\.\d{2}\.\d{4}", text))
|
||||
dates_parsed: list[date] = []
|
||||
for m in date_matches:
|
||||
d = parse_dmy(m.group(0))
|
||||
if d is not None:
|
||||
dates_parsed.append(d)
|
||||
publish_date = dates_parsed[0] if dates_parsed else None
|
||||
removed_date = dates_parsed[1] if len(dates_parsed) >= 2 else None
|
||||
expo_m = RE_ITEM_EXPOSURE.search(text)
|
||||
exposure_days = int(expo_m.group(1)) if expo_m else None
|
||||
|
||||
|
|
@ -304,6 +331,7 @@ class YandexValuationScraper(BaseScraper):
|
|||
last_price=last_price,
|
||||
last_price_per_m2=last_ppm2,
|
||||
publish_date=publish_date,
|
||||
removed_date=removed_date,
|
||||
exposure_days=exposure_days,
|
||||
status=status,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -20,9 +20,7 @@ from app.services.scrapers.yandex_valuation import (
|
|||
# Fixture helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_HOUSE_META_BLOCK = (
|
||||
"12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт"
|
||||
)
|
||||
_HOUSE_META_BLOCK = "12 объектов Дом 1981 года 9 этажей Панельное здание 2,50 м потолки Лифт"
|
||||
|
||||
_ITEM_1 = (
|
||||
"2-комнатная квартира 45,3 м² 4 этаж "
|
||||
|
|
@ -52,9 +50,7 @@ def _make_full_html(body_content: str) -> str:
|
|||
return f"<html><body>{body_content}</body></html>"
|
||||
|
||||
|
||||
FULL_FIXTURE_HTML = _make_full_html(
|
||||
f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}"
|
||||
)
|
||||
FULL_FIXTURE_HTML = _make_full_html(f"{_HOUSE_META_BLOCK}\n{_ITEM_1}\n{_ITEM_2}\n{_ITEM_3}")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
|
@ -155,9 +151,7 @@ def test_parse_items_chunked_dedup():
|
|||
duplicate_block = f"{_ITEM_1}\n{_ITEM_1}"
|
||||
items = YandexValuationScraper._parse_items_from_chunked_text(duplicate_block)
|
||||
# Both items reference 15.03.2023 + 45.3 m2 + floor 4 — must dedup to 1
|
||||
matching = [
|
||||
i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)
|
||||
]
|
||||
matching = [i for i in items if i.area_m2 == 45.3 and i.publish_date == date(2023, 3, 15)]
|
||||
assert len(matching) == 1
|
||||
|
||||
|
||||
|
|
@ -240,3 +234,106 @@ def test_parse_ceiling_height_comma():
|
|||
text = "Дом 1999 года 5 этажей Кирпичное здание 2,70 м потолки Лифт"
|
||||
meta = YandexValuationScraper._parse_house_meta(text)
|
||||
assert meta.ceiling_height == 2.7
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Regression tests — parser fixes (2026-05-24)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_item_extracts_removed_date():
|
||||
"""Two dates in chunk → first = publish_date, second = removed_date."""
|
||||
text = "2-комнатная квартира 50,0 м² 3 этаж 10.05.2024 В экспозиции 30 дней 09.06.2024"
|
||||
item = YandexValuationScraper._parse_item_text(text)
|
||||
assert item is not None
|
||||
assert item.publish_date == date(2024, 5, 10)
|
||||
assert item.removed_date == date(2024, 6, 9)
|
||||
|
||||
|
||||
def test_parse_item_in_sale_has_no_removed_date():
|
||||
"""Single date + 'В продаже' → removed_date is None."""
|
||||
text = "1-комнатная квартира 40 м² 5 этаж 10.05.2024 В экспозиции 30 дней В продаже"
|
||||
item = YandexValuationScraper._parse_item_text(text)
|
||||
assert item is not None
|
||||
assert item.publish_date == date(2024, 5, 10)
|
||||
assert item.removed_date is None
|
||||
|
||||
|
||||
def test_total_floors_extracted_from_dom_meta_not_items():
|
||||
"""Regression: 'N этаж' inside each item must NOT be matched as dom total_floors.
|
||||
Real Yandex page has 'M этажей' (plural) in dom-meta and 'N этаж' (singular) per item.
|
||||
"""
|
||||
text = (
|
||||
"Дом 2025 года Панорама 25 этажей Монолитное здание 2,7 м потолки Лифт "
|
||||
"1-комнатная 40 м² 3 этаж 10.01.2026 В экспозиции 5 дней В продаже "
|
||||
"2-комнатная 55 м² 17 этаж 05.01.2026 В экспозиции 10 дней В продаже"
|
||||
)
|
||||
meta = YandexValuationScraper._parse_house_meta(text)
|
||||
assert meta.year_built == 2025
|
||||
assert meta.total_floors == 25
|
||||
assert meta.house_type == "monolith"
|
||||
|
||||
|
||||
def test_parse_normalizes_nbsp_for_price_per_m2():
|
||||
"""NBSP (\\xa0) in price tokens must not break price_per_m2 regex.
|
||||
Real Yandex SSR HTML uses NBSP between digit groups and before ₽.
|
||||
"""
|
||||
html = (
|
||||
"<html><body>"
|
||||
"Дом 2020 года 16 этажей Монолитное здание "
|
||||
"2-комнатная 50,0 м² 3 этаж "
|
||||
"10.05.2024 В экспозиции 30 дней В продаже "
|
||||
"5\xa0000\xa0000 ₽ 100\xa0000 ₽ за м² "
|
||||
"4\xa0900\xa0000 ₽ 98\xa0000 ₽ за м² "
|
||||
"</body></html>"
|
||||
)
|
||||
scraper = YandexValuationScraper()
|
||||
result = scraper.parse(
|
||||
html,
|
||||
address="test",
|
||||
offer_category="APARTMENT",
|
||||
offer_type="SELL",
|
||||
page=1,
|
||||
source_url="http://test/",
|
||||
)
|
||||
assert len(result.history_items) >= 1
|
||||
item = result.history_items[0]
|
||||
assert item.start_price == 5_000_000
|
||||
assert item.start_price_per_m2 == 100_000
|
||||
|
||||
|
||||
def test_validate_match_full_match():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=2025, total_floors=25)
|
||||
assert meta.validate_match(2025, 25) == 1.0
|
||||
|
||||
|
||||
def test_validate_match_year_mismatch():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=1984, total_floors=9)
|
||||
assert meta.validate_match(2025, 25) == 0.0
|
||||
|
||||
|
||||
def test_validate_match_partial():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=2025, total_floors=10)
|
||||
# year matches, floors mismatch (>1 tolerance) → 0.5
|
||||
assert meta.validate_match(2025, 25) == 0.5
|
||||
|
||||
|
||||
def test_validate_match_tolerance_one_year():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=2024, total_floors=25)
|
||||
# ±1 year tolerance
|
||||
assert meta.validate_match(2025, 25) == 1.0
|
||||
|
||||
|
||||
def test_validate_match_no_expected_returns_one():
|
||||
from app.services.scrapers.yandex_valuation import ValuationHouseMeta
|
||||
|
||||
meta = ValuationHouseMeta(year_built=None, total_floors=None)
|
||||
assert meta.validate_match(None, None) == 1.0
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue