fix(yandex-valuation): cap area_m2 sanity (drop >10000 or <=0)

Live observation: sweep on "Базовый переулок, 52" produced area_m2=2_025_106.7
(concatenated digits across DOM boundaries) -> psycopg.NumericValueOutOfRange on
INSERT, lost entire batch of 30 items from page=1.

Cap at 10_000 m2 (residential ceiling) and drop <=0. Keeps the rest of the
item (price, dates, status) so the row still persists with area_m2=NULL --
better than losing the whole batch.
This commit is contained in:
lekss361 2026-05-24 18:09:12 +03:00
parent ee2262d605
commit 4939d04360
2 changed files with 58 additions and 1 deletions

View file

@ -281,7 +281,17 @@ class YandexValuationScraper(BaseScraper):
# area + rooms # area + rooms
area_m = RE_ITEM_AREA.search(text) area_m = RE_ITEM_AREA.search(text)
area_m2 = float(area_m.group(1).replace(",", ".")) if area_m else None area_m2: float | None = float(area_m.group(1).replace(",", ".")) if area_m else None
# Sanity cap — Yandex page text sometimes concatenates digits across DOM
# boundaries (e.g. "2025\xa0106,7 м²" → 2_025_106.7). Flats over 10_000 m²
# are impossible; DB column is NUMERIC(8,2) which overflows at 10^6.
# Drop the value rather than block the whole save batch.
if area_m2 is not None and (area_m2 > 10_000 or area_m2 <= 0):
logger.warning(
"yandex_valuation: dropping nonsensical area_m2=%s from item chunk",
area_m2,
)
area_m2 = None
rooms: int | None = None rooms: int | None = None
if RE_ITEM_STUDIO.search(text): if RE_ITEM_STUDIO.search(text):

View file

@ -337,3 +337,50 @@ def test_validate_match_no_expected_returns_one():
meta = ValuationHouseMeta(year_built=None, total_floors=None) meta = ValuationHouseMeta(year_built=None, total_floors=None)
assert meta.validate_match(None, None) == 1.0 assert meta.validate_match(None, None) == 1.0
# ---------------------------------------------------------------------------
# Sanity cap regression — area_m2 overflow (2026-05-24)
# ---------------------------------------------------------------------------
def test_parse_item_drops_overflow_area():
"""Regression: area_m2 > 10_000 (e.g. concatenated digits) must be set None,
not blow up the save batch. Dates and exposure_days should survive.
The live bug: chunk-text fallback concatenated digits across a DOM boundary,
producing area_m2=2_025_106.7 psycopg.NumericValueOutOfRange on INSERT.
The cap drops area_m2 to None so the row still saves with area NULL.
"""
# Bare concatenated digits (no whitespace between groups) — reproduction of
# what the chunked-text path emitted in production. Price is placed after
# the area token so the item stays valid (area_m2 dropped → start_price keeps it).
text = (
"3-комнатная 2025106,7 м² 25 этаж "
"01.07.2024 5 500 000 ₽ В экспозиции 514 дней 26.11.2025"
)
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
assert item.area_m2 is None, f"expected None for absurd area, got {item.area_m2}"
assert item.publish_date == date(2024, 7, 1)
assert item.removed_date == date(2025, 11, 26)
def test_parse_item_keeps_normal_area():
"""Normal residential area must pass through unchanged."""
text = (
"2-комнатная 58,2 м² 3 этаж "
"10.05.2024 9 300 000 ₽ 159 932 ₽ за м² "
"9 300 000 ₽ 159 932 ₽ за м² "
"В экспозиции 30 дней В продаже"
)
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
assert item.area_m2 == 58.2
def test_parse_item_drops_zero_area():
text = "1-комнатная 0 м² 3 этаж 10.05.2024 5 000 000 ₽ В продаже"
item = YandexValuationScraper._parse_item_text(text)
assert item is not None
assert item.area_m2 is None