From 490bc8ca5d3ac5d5a7ac0ed4c4a6f858a35f3afd Mon Sep 17 00:00:00 2001
From: lekss361 <lekss361@gendsgn.local>
Date: Sun, 24 May 2026 15:37:48 +0000
Subject: [PATCH] fix(yandex-valuation): tighten area regex + chronological
 date ordering (#541)

---
 .../app/services/scrapers/yandex_valuation.py | 17 ++++-
 .../backend/tests/test_yandex_valuation.py    | 64 +++++++++++++++++++
 2 files changed, 78 insertions(+), 3 deletions(-)

diff --git a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py
index 64e75638..10fc3316 100644
--- a/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py
+++ b/tradein-mvp/backend/app/services/scrapers/yandex_valuation.py
@@ -120,7 +120,10 @@ RE_FLOORS = re.compile(r"(\d+)\s+этажей", re.IGNORECASE)
 RE_CEILING = re.compile(r"([\d,]+)\s*м\s+потолки", re.IGNORECASE)
 RE_TOTAL_OBJECTS = re.compile(r"(\d+)\s+объект", re.IGNORECASE)
 
-RE_ITEM_AREA = re.compile(r"(\d+[.,]?\d*)\s*м²")
+# Negative lookbehind on digit/comma/dot prevents matching the trailing digits of
+# adjacent tokens — e.g. publish year "2024" right before "42,6 м²" must NOT
+# yield "20242,6". Cap whole-number part at 4 digits and decimal part at 2.
+RE_ITEM_AREA = re.compile(r"(?<![\d.,])(\d{1,4}(?:[.,]\d{1,2})?)\s*м²")
 RE_ITEM_ROOMS = re.compile(r"(\d+)\s*-\s*комнатн", re.IGNORECASE)
 RE_ITEM_STUDIO = re.compile(r"студи[яюй]", re.IGNORECASE)
 RE_ITEM_FLOOR = re.compile(r"(\d+)\s*этаж", re.IGNORECASE)
@@ -320,8 +323,16 @@ class YandexValuationScraper(BaseScraper):
             d = parse_dmy(m.group(0))
             if d is not None:
                 dates_parsed.append(d)
-        publish_date = dates_parsed[0] if dates_parsed else None
-        removed_date = dates_parsed[1] if len(dates_parsed) >= 2 else None
+        # Sort dates chronologically — chunked-text scan returns them in page-text
+        # order, but semantically publish_date is earliest and removed_date is
+        # latest. Without this, ~1% of rows on prod show removed < publish.
+        if dates_parsed:
+            dates_sorted = sorted(d for d in dates_parsed if d is not None)
+            publish_date = dates_sorted[0] if dates_sorted else None
+            removed_date = dates_sorted[1] if len(dates_sorted) >= 2 else None
+        else:
+            publish_date = None
+            removed_date = None
         expo_m = RE_ITEM_EXPOSURE.search(text)
         exposure_days = int(expo_m.group(1)) if expo_m else None
 
diff --git a/tradein-mvp/backend/tests/test_yandex_valuation.py b/tradein-mvp/backend/tests/test_yandex_valuation.py
index 95a07ace..316da0ee 100644
--- a/tradein-mvp/backend/tests/test_yandex_valuation.py
+++ b/tradein-mvp/backend/tests/test_yandex_valuation.py
@@ -384,3 +384,67 @@ def test_parse_item_drops_zero_area():
     item = YandexValuationScraper._parse_item_text(text)
     assert item is not None
     assert item.area_m2 is None
+
+
+# ---------------------------------------------------------------------------
+# Regex tightening + chronological date ordering (2026-05-24)
+# ---------------------------------------------------------------------------
+
+
+def test_area_regex_rejects_year_concat():
+    """RE_ITEM_AREA must NOT match digits glued onto a preceding number.
+    Real bug on prod: 'опубликовано 01.07.2024' + '52,2 м²' rendered into chunked
+    text as '202452,2 м²' → matched as 202452.2.
+    """
+    text = "2-комнатная 202452,2 м²  3 этаж  01.07.2024  9 300 000 ₽  В продаже"
+    item = YandexValuationScraper._parse_item_text(text)
+    # The "202452,2" token has digits jammed before it (no separator), so the
+    # tightened regex must NOT match it.
+    assert item is None or item.area_m2 is None, (
+        f"expected no area match for concat token, got {item.area_m2 if item else 'None item'}"
+    )
+
+
+def test_area_regex_accepts_isolated_token():
+    """Normal '58,2 м²' surrounded by whitespace must still be parsed."""
+    text = "2-комн 58,2 м²  3 этаж  10.05.2024  9 300 000 ₽  В продаже"
+    item = YandexValuationScraper._parse_item_text(text)
+    assert item is not None
+    assert item.area_m2 == 58.2
+
+
+def test_area_regex_capped_at_4_digits():
+    """Anything > 9999.99 m² must not match (real flats are <500 m²)."""
+    text = "квартира 12345 м²  3 этаж  10.05.2024  5 000 000 ₽  В продаже"
+    item = YandexValuationScraper._parse_item_text(text)
+    # 5-digit area shouldn't match; remaining fields still salvageable
+    assert item is None or item.area_m2 is None
+
+
+def test_dates_sorted_publish_before_removed():
+    """Even if page-text emits removed-date first, parser must put earlier
+    date in publish_date and later in removed_date.
+    """
+    # Reversed order: removed appears in text before publish
+    text = (
+        "2-комн 50,0 м²  3 этаж  "
+        "13.08.2025  В экспозиции 156 дней  12.02.2026  "
+        "8 000 000 ₽  В продаже"
+    )
+    item = YandexValuationScraper._parse_item_text(text)
+    assert item is not None
+    from datetime import date as _d
+
+    assert item.publish_date == _d(2025, 8, 13)
+    assert item.removed_date == _d(2026, 2, 12)
+
+
+def test_single_date_is_publish_only():
+    """One date → publish_date set, removed_date None."""
+    text = "1-комн 40,0 м²  3 этаж  15.03.2024  В экспозиции 30 дней  В продаже"
+    item = YandexValuationScraper._parse_item_text(text)
+    assert item is not None
+    from datetime import date as _d
+
+    assert item.publish_date == _d(2024, 3, 15)
+    assert item.removed_date is None