fix(tradein): word-boundary street match + cian unescape robustness (Refs #769 A1+D) (#798)

Co-authored-by: bot-backend <bot-backend@gendsgn.local> Co-committed-by: bot-backend <bot-backend@gendsgn.local>
2026-05-30 17:18:33 +00:00 · 2026-05-30 17:18:33 +00:00 · 4c275e04ea
commit 4c275e04ea
parent 87a5de0cae
5 changed files with 123 additions and 25 deletions
--- a/tradein-mvp/backend/app/api/v1/trade_in.py
+++ b/tradein-mvp/backend/app/api/v1/trade_in.py
@ -1478,6 +1478,7 @@ def get_street_deals(
            FROM deals
            WHERE source = 'rosreestr'
              AND address ILIKE :street_pattern
+              AND address ~* :street_regex
              AND rooms = CAST(:rooms AS integer)
              AND area_m2 BETWEEN :area_min AND :area_max
              AND deal_date > NOW() - (CAST(:period_months AS integer) || ' months')::interval
@ -1487,6 +1488,7 @@ def get_street_deals(
            ),
            {
                "street_pattern": "%" + street_name + "%",
+                "street_regex": r"\m" + street_name + r"\M",
                "rooms": rooms,
                "area_min": area_min,
                "area_max": area_max,
--- a/tradein-mvp/backend/app/services/estimator.py
+++ b/tradein-mvp/backend/app/services/estimator.py
@ -821,6 +821,7 @@ def _fetch_dkp_corridor(
                    FROM deals
                    WHERE source = 'rosreestr'
                      AND address ILIKE :street_pattern
+                      AND address ~* :street_regex
                      AND rooms = CAST(:rooms AS integer)
                      AND area_m2 BETWEEN :area_min AND :area_max
                      AND deal_date > NOW()
@ -832,6 +833,7 @@ def _fetch_dkp_corridor(
                ),
                {
                    "street_pattern": "%" + street_name + "%",
+                    "street_regex": r"\m" + street_name + r"\M",
                    "rooms": rooms,
                    "area_min": area_min,
                    "area_max": area_max,
--- a/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py
+++ b/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py
@ -3,6 +3,7 @@
 Pattern: window._cianConfig['<mfe>'].push({key:..., value:..., priority:..., filter:...})
 Used by SERP / detail / newbuilding / valuation scrapers.
 """
+
 import json
 import logging
 import re
@ -45,23 +46,24 @@ def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, A
            pass

        # Strategy 2: JSON.parse("escaped string")
-        m_jp = re.match(r'JSON\.parse\(\s*"(.+?)"\s*\)\s*$', value_str, re.DOTALL)
+        # Use json.loads on the outer quoted string to unescape atomically — avoids
+        # order-dependent manual .replace() chains that break when descriptions contain
+        # backslashes (e.g. '\\' in listing text corrupts the rest of the parse).
+        m_jp = re.match(r'JSON\.parse\(\s*("(?:[^"\\]|\\.)*")\s*\)\s*$', value_str, re.DOTALL)
        if m_jp:
-            escaped = m_jp.group(1)
-            unescaped = (
-                escaped
-                .replace('\\"', '"')
-                .replace('\\\\', '\\')
-                .replace('\\n', '\n')
-                .replace('\\/', '/')
-            )
+            outer_json_str = m_jp.group(1)  # the full "..." including surrounding quotes
            try:
-                return json.loads(unescaped)
+                inner = json.loads(outer_json_str)  # inner is now a Python str
            except json.JSONDecodeError as exc:
                logger.debug(
-                    "JSON.parse() unescape failed for mfe=%s key=%s: %s", mfe, key, exc
+                    "JSON.parse() outer unescape failed for mfe=%s key=%s: %s", mfe, key, exc
                )
                continue
+            try:
+                return json.loads(inner)
+            except json.JSONDecodeError as exc:
+                logger.debug("JSON.parse() inner parse failed for mfe=%s key=%s: %s", mfe, key, exc)
+                continue

        # Strategy 3: raw JS object literal — try demjson3 (optional dep)
        try:
--- a/tradein-mvp/backend/tests/test_cian_state_parser.py
+++ b/tradein-mvp/backend/tests/test_cian_state_parser.py
@ -1,10 +1,10 @@
 """Tests for cian_state_parser."""
-import pytest
+
 from app.services.scrapers.cian_state_parser import extract_state


 def test_extract_state_simple_json():
-    html = '''
+    html = """
    <html><body>
    <script>
    window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];
@ -15,32 +15,64 @@ def test_extract_state_simple_json():
    });
    </script>
    </body></html>
-    '''
-    state = extract_state(html, mfe='frontend-serp', key='initialState')
+    """
+    state = extract_state(html, mfe="frontend-serp", key="initialState")
    assert state is not None
-    assert state['results']['offers'][0]['id'] == 1
+    assert state["results"]["offers"][0]["id"] == 1


 def test_extract_state_json_parse_form():
-    html = r'''
+    html = r"""
    window._cianConfig['frontend-offer-card'].push({
        key: 'defaultState',
        value: JSON.parse("{\"offerData\":{\"offer\":{\"cianId\":123}}}"),
        priority: 1
    });
-    '''
-    state = extract_state(html, mfe='frontend-offer-card', key='defaultState')
+    """
+    state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
    assert state is not None
-    assert state['offerData']['offer']['cianId'] == 123
+    assert state["offerData"]["offer"]["cianId"] == 123


 def test_extract_state_missing_returns_none():
    html = "<html></html>"
-    assert extract_state(html, mfe='frontend-serp') is None
+    assert extract_state(html, mfe="frontend-serp") is None


 def test_extract_state_wrong_mfe_returns_none():
-    html = '''
+    html = """
    window._cianConfig['other-mfe'].push({key: 'initialState', value: {}, priority: 1});
-    '''
-    assert extract_state(html, mfe='frontend-serp', key='initialState') is None
+    """
+    assert extract_state(html, mfe="frontend-serp", key="initialState") is None
+
+
+def test_extract_state_backslash_in_description() -> None:
+    """Backslash in a listing description must not break page parsing (Part D fix).
+
+    The hand-rolled .replace() chain was order-dependent: '\\\\' → '\\' after
+    '\\"' → '"' left stray backslashes that broke json.loads on the inner payload.
+    The fix uses json.loads on the outer quoted string atomically.
+    """
+    # Build a JSON payload where the description contains a literal backslash.
+    # In the wire format (JSON.parse("...")), the inner JSON is double-escaped:
+    #   inner JSON:  {"desc": "с \\ слешем"}
+    #   outer string: "{\"desc\": \"с \\\\ слешем\"}"
+    import json
+
+    inner_payload = {"offerData": {"offer": {"cianId": 42, "description": r"описание с \ слешем"}}}
+    inner_json_str = json.dumps(inner_payload, ensure_ascii=False)
+    # Simulate what Cian emits: JSON.parse("<double-escaped inner>")
+    outer_escaped = json.dumps(inner_json_str)  # produces "\"...\"" with double escaping
+    html = f"""
+    window._cianConfig['frontend-offer-card'].push({{
+        key: 'defaultState',
+        value: JSON.parse({outer_escaped}),
+        priority: 1
+    }});
+    """
+    state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
+    assert state is not None, "extract_state returned None — page parse failed"
+    assert state["offerData"]["offer"]["cianId"] == 42
+    assert (
+        "\\" in state["offerData"]["offer"]["description"]
+    ), "Backslash in description was lost during unescape"
--- a/tradein-mvp/backend/tests/test_street_deals_endpoint.py
+++ b/tradein-mvp/backend/tests/test_street_deals_endpoint.py
@ -233,10 +233,70 @@ def test_street_deals_aggregates_correctly(trade_in_app: FastAPI) -> None:
    assert data["street"] == "Космонавтов"
    assert data["count"] == 5
    assert data["median_price_per_m2"] == 100_000
-    assert data["median_price_rub"] == 5_000_000    # 100k * 50m²
+    assert data["median_price_rub"] == 5_000_000  # 100k * 50m²
    assert data["range_low_rub"] == 4_000_000
    assert data["range_high_rub"] == 6_000_000
    # All 5 deals returned (top-10 threshold)
    assert len(data["deals"]) == 5
    # DB rows already ordered by deal_date DESC — first is most recent (2025-10-01)
    assert data["deals"][0]["price_rub"] == 6_000_000
+
+
+# ── Test: word-boundary street match (Part A1 fix) ───────────────────────────
+
+
+def test_street_regex_word_boundary_no_false_positive() -> None:
+    """«Мира» must NOT match «улица Адмирала Макарова» (contains 'мира' as substring).
+
+    We verify the pattern construction directly using Python re (\\b is equivalent
+    to Postgres \\m/\\M for word-boundary checks on these inputs).
+    """
+    import re
+
+    street = "Мира"
+    pattern = r"\b" + re.escape(street) + r"\b"
+
+    # Must NOT match — 'мира' appears inside 'Макарова' substring check skipped
+    # but more critically — 'мира' appears inside 'Адмирала Макарова'
+    false_positive_addr = "улица Адмирала Макарова"
+    assert not re.search(
+        pattern, false_positive_addr, re.IGNORECASE
+    ), f"Pattern {pattern!r} should NOT match {false_positive_addr!r}"
+
+    # Must match — exact word
+    true_positive_addr = "улица Мира 5"
+    assert re.search(
+        pattern, true_positive_addr, re.IGNORECASE
+    ), f"Pattern {pattern!r} should match {true_positive_addr!r}"
+
+
+def test_street_regex_param_passed_to_db(trade_in_app: FastAPI) -> None:
+    """Endpoint passes street_regex with word-boundary markers to DB execute()."""
+    db_mock = _make_db_mock([])
+
+    from app.core.db import get_db
+
+    def _override():
+        yield db_mock
+
+    trade_in_app.dependency_overrides[get_db] = _override
+
+    client = TestClient(trade_in_app)
+    client.get(
+        "/api/v1/trade-in/street-deals",
+        params={
+            "address": "г. Екатеринбург, ул. Мира, 10",
+            "area_m2": 50.0,
+            "rooms": 2,
+        },
+    )
+
+    assert db_mock.execute.called, "db.execute should have been called"
+    call_args = db_mock.execute.call_args
+    params = call_args[0][1] if len(call_args[0]) > 1 else call_args[1].get("parameters", {})
+    assert "street_regex" in params, f"street_regex not in params: {params}"
+    regex_val = params["street_regex"]
+    # Must contain word-boundary anchors
+    assert (
+        r"\m" in regex_val or r"\b" in regex_val or regex_val.startswith(r"\m")
+    ), f"Expected word-boundary in regex, got: {regex_val!r}"