fix(yandex-valuation): cap area_m2 sanity (drop >10000 or ≤0) #538
2 changed files with 58 additions and 1 deletions
|
|
@ -281,7 +281,17 @@ class YandexValuationScraper(BaseScraper):
|
|||
|
||||
# area + rooms
|
||||
area_m = RE_ITEM_AREA.search(text)
|
||||
area_m2 = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None
|
||||
# Sanity cap — Yandex page text sometimes concatenates digits across DOM
|
||||
# boundaries (e.g. "2025\xa0106,7 м²" → 2_025_106.7). Flats over 10_000 m²
|
||||
# are impossible; DB column is NUMERIC(8,2) which overflows at 10^6.
|
||||
# Drop the value rather than block the whole save batch.
|
||||
if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0):
|
||||
logger.warning(
|
||||
"yandex_valuation: dropping nonsensical area_m2=%s from item chunk",
|
||||
area_m2,
|
||||
)
|
||||
area_m2 = None
|
||||
|
||||
rooms: int | None = None
|
||||
if RE_ITEM_STUDIO.search(text):
|
||||
|
|
|
|||
|
|
@ -337,3 +337,50 @@ def test_validate_match_no_expected_returns_one():
|
|||
|
||||
meta = ValuationHouseMeta(year_built=None, total_floors=None)
|
||||
assert meta.validate_match(None, None) == 1.0
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sanity cap regression — area_m2 overflow (2026-05-24)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_parse_item_drops_overflow_area():
|
||||
"""Regression: area_m2 > 10_000 (e.g. concatenated digits) must be set None,
|
||||
not blow up the save batch. Dates and exposure_days should survive.
|
||||
|
||||
The live bug: chunk-text fallback concatenated digits across a DOM boundary,
|
||||
producing area_m2=2_025_106.7 → psycopg.NumericValueOutOfRange on INSERT.
|
||||
The cap drops area_m2 to None so the row still saves with area NULL.
|
||||
"""
|
||||
# Bare concatenated digits (no whitespace between groups) — reproduction of
|
||||
# what the chunked-text path emitted in production. Price is placed after
|
||||
# the area token so the item stays valid (area_m2 dropped → start_price keeps it).
|
||||
text = (
|
||||
"3-комнатная 2025106,7 м² 25 этаж "
|
||||
"01.07.2024 5 500 000 ₽ В экспозиции 514 дней 26.11.2025"
|
||||
)
|
||||
item = YandexValuationScraper._parse_item_text(text)
|
||||
assert item is not None
|
||||
assert item.area_m2 is None, f"expected None for absurd area, got {item.area_m2}"
|
||||
assert item.publish_date == date(2024, 7, 1)
|
||||
assert item.removed_date == date(2025, 11, 26)
|
||||
|
||||
|
||||
def test_parse_item_keeps_normal_area():
|
||||
"""Normal residential area must pass through unchanged."""
|
||||
text = (
|
||||
"2-комнатная 58,2 м² 3 этаж "
|
||||
"10.05.2024 9 300 000 ₽ 159 932 ₽ за м² "
|
||||
"9 300 000 ₽ 159 932 ₽ за м² "
|
||||
"В экспозиции 30 дней В продаже"
|
||||
)
|
||||
item = YandexValuationScraper._parse_item_text(text)
|
||||
assert item is not None
|
||||
assert item.area_m2 == 58.2
|
||||
|
||||
|
||||
def test_parse_item_drops_zero_area():
|
||||
text = "1-комнатная 0 м² 3 этаж 10.05.2024 5 000 000 ₽ В продаже"
|
||||
item = YandexValuationScraper._parse_item_text(text)
|
||||
assert item is not None
|
||||
assert item.area_m2 is None
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue