diff --git a/tradein-mvp/backend/app/api/v1/trade_in.py b/tradein-mvp/backend/app/api/v1/trade_in.py index ac67695b..bcf8f141 100644 --- a/tradein-mvp/backend/app/api/v1/trade_in.py +++ b/tradein-mvp/backend/app/api/v1/trade_in.py @@ -1478,6 +1478,7 @@ def get_street_deals( FROM deals WHERE source = 'rosreestr' AND address ILIKE :street_pattern + AND address ~* :street_regex AND rooms = CAST(:rooms AS integer) AND area_m2 BETWEEN :area_min AND :area_max AND deal_date > NOW() - (CAST(:period_months AS integer) || ' months')::interval @@ -1487,6 +1488,7 @@ def get_street_deals( ), { "street_pattern": "%" + street_name + "%", + "street_regex": r"\m" + street_name + r"\M", "rooms": rooms, "area_min": area_min, "area_max": area_max, diff --git a/tradein-mvp/backend/app/services/estimator.py b/tradein-mvp/backend/app/services/estimator.py index 7542686f..d0074c7f 100644 --- a/tradein-mvp/backend/app/services/estimator.py +++ b/tradein-mvp/backend/app/services/estimator.py @@ -821,6 +821,7 @@ def _fetch_dkp_corridor( FROM deals WHERE source = 'rosreestr' AND address ILIKE :street_pattern + AND address ~* :street_regex AND rooms = CAST(:rooms AS integer) AND area_m2 BETWEEN :area_min AND :area_max AND deal_date > NOW() @@ -832,6 +833,7 @@ def _fetch_dkp_corridor( ), { "street_pattern": "%" + street_name + "%", + "street_regex": r"\m" + street_name + r"\M", "rooms": rooms, "area_min": area_min, "area_max": area_max, diff --git a/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py b/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py index 429aa962..f0fef80c 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py @@ -3,6 +3,7 @@ Pattern: window._cianConfig[''].push({key:..., value:..., priority:..., filter:...}) Used by SERP / detail / newbuilding / valuation scrapers. """ + import json import logging import re @@ -45,23 +46,24 @@ def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, A pass # Strategy 2: JSON.parse("escaped string") - m_jp = re.match(r'JSON\.parse\(\s*"(.+?)"\s*\)\s*$', value_str, re.DOTALL) + # Use json.loads on the outer quoted string to unescape atomically — avoids + # order-dependent manual .replace() chains that break when descriptions contain + # backslashes (e.g. '\\' in listing text corrupts the rest of the parse). + m_jp = re.match(r'JSON\.parse\(\s*("(?:[^"\\]|\\.)*")\s*\)\s*$', value_str, re.DOTALL) if m_jp: - escaped = m_jp.group(1) - unescaped = ( - escaped - .replace('\\"', '"') - .replace('\\\\', '\\') - .replace('\\n', '\n') - .replace('\\/', '/') - ) + outer_json_str = m_jp.group(1) # the full "..." including surrounding quotes try: - return json.loads(unescaped) + inner = json.loads(outer_json_str) # inner is now a Python str except json.JSONDecodeError as exc: logger.debug( - "JSON.parse() unescape failed for mfe=%s key=%s: %s", mfe, key, exc + "JSON.parse() outer unescape failed for mfe=%s key=%s: %s", mfe, key, exc ) continue + try: + return json.loads(inner) + except json.JSONDecodeError as exc: + logger.debug("JSON.parse() inner parse failed for mfe=%s key=%s: %s", mfe, key, exc) + continue # Strategy 3: raw JS object literal — try demjson3 (optional dep) try: diff --git a/tradein-mvp/backend/tests/test_cian_state_parser.py b/tradein-mvp/backend/tests/test_cian_state_parser.py index 8ee05321..2c07035c 100644 --- a/tradein-mvp/backend/tests/test_cian_state_parser.py +++ b/tradein-mvp/backend/tests/test_cian_state_parser.py @@ -1,10 +1,10 @@ """Tests for cian_state_parser.""" -import pytest + from app.services.scrapers.cian_state_parser import extract_state def test_extract_state_simple_json(): - html = ''' + html = """ - ''' - state = extract_state(html, mfe='frontend-serp', key='initialState') + """ + state = extract_state(html, mfe="frontend-serp", key="initialState") assert state is not None - assert state['results']['offers'][0]['id'] == 1 + assert state["results"]["offers"][0]["id"] == 1 def test_extract_state_json_parse_form(): - html = r''' + html = r""" window._cianConfig['frontend-offer-card'].push({ key: 'defaultState', value: JSON.parse("{\"offerData\":{\"offer\":{\"cianId\":123}}}"), priority: 1 }); - ''' - state = extract_state(html, mfe='frontend-offer-card', key='defaultState') + """ + state = extract_state(html, mfe="frontend-offer-card", key="defaultState") assert state is not None - assert state['offerData']['offer']['cianId'] == 123 + assert state["offerData"]["offer"]["cianId"] == 123 def test_extract_state_missing_returns_none(): html = "" - assert extract_state(html, mfe='frontend-serp') is None + assert extract_state(html, mfe="frontend-serp") is None def test_extract_state_wrong_mfe_returns_none(): - html = ''' + html = """ window._cianConfig['other-mfe'].push({key: 'initialState', value: {}, priority: 1}); - ''' - assert extract_state(html, mfe='frontend-serp', key='initialState') is None + """ + assert extract_state(html, mfe="frontend-serp", key="initialState") is None + + +def test_extract_state_backslash_in_description() -> None: + """Backslash in a listing description must not break page parsing (Part D fix). + + The hand-rolled .replace() chain was order-dependent: '\\\\' → '\\' after + '\\"' → '"' left stray backslashes that broke json.loads on the inner payload. + The fix uses json.loads on the outer quoted string atomically. + """ + # Build a JSON payload where the description contains a literal backslash. + # In the wire format (JSON.parse("...")), the inner JSON is double-escaped: + # inner JSON: {"desc": "с \\ слешем"} + # outer string: "{\"desc\": \"с \\\\ слешем\"}" + import json + + inner_payload = {"offerData": {"offer": {"cianId": 42, "description": r"описание с \ слешем"}}} + inner_json_str = json.dumps(inner_payload, ensure_ascii=False) + # Simulate what Cian emits: JSON.parse("") + outer_escaped = json.dumps(inner_json_str) # produces "\"...\"" with double escaping + html = f""" + window._cianConfig['frontend-offer-card'].push({{ + key: 'defaultState', + value: JSON.parse({outer_escaped}), + priority: 1 + }}); + """ + state = extract_state(html, mfe="frontend-offer-card", key="defaultState") + assert state is not None, "extract_state returned None — page parse failed" + assert state["offerData"]["offer"]["cianId"] == 42 + assert ( + "\\" in state["offerData"]["offer"]["description"] + ), "Backslash in description was lost during unescape" diff --git a/tradein-mvp/backend/tests/test_street_deals_endpoint.py b/tradein-mvp/backend/tests/test_street_deals_endpoint.py index e1bdbf6e..c1ea1249 100644 --- a/tradein-mvp/backend/tests/test_street_deals_endpoint.py +++ b/tradein-mvp/backend/tests/test_street_deals_endpoint.py @@ -233,10 +233,70 @@ def test_street_deals_aggregates_correctly(trade_in_app: FastAPI) -> None: assert data["street"] == "Космонавтов" assert data["count"] == 5 assert data["median_price_per_m2"] == 100_000 - assert data["median_price_rub"] == 5_000_000 # 100k * 50m² + assert data["median_price_rub"] == 5_000_000 # 100k * 50m² assert data["range_low_rub"] == 4_000_000 assert data["range_high_rub"] == 6_000_000 # All 5 deals returned (top-10 threshold) assert len(data["deals"]) == 5 # DB rows already ordered by deal_date DESC — first is most recent (2025-10-01) assert data["deals"][0]["price_rub"] == 6_000_000 + + +# ── Test: word-boundary street match (Part A1 fix) ─────────────────────────── + + +def test_street_regex_word_boundary_no_false_positive() -> None: + """«Мира» must NOT match «улица Адмирала Макарова» (contains 'мира' as substring). + + We verify the pattern construction directly using Python re (\\b is equivalent + to Postgres \\m/\\M for word-boundary checks on these inputs). + """ + import re + + street = "Мира" + pattern = r"\b" + re.escape(street) + r"\b" + + # Must NOT match — 'мира' appears inside 'Макарова' substring check skipped + # but more critically — 'мира' appears inside 'Адмирала Макарова' + false_positive_addr = "улица Адмирала Макарова" + assert not re.search( + pattern, false_positive_addr, re.IGNORECASE + ), f"Pattern {pattern!r} should NOT match {false_positive_addr!r}" + + # Must match — exact word + true_positive_addr = "улица Мира 5" + assert re.search( + pattern, true_positive_addr, re.IGNORECASE + ), f"Pattern {pattern!r} should match {true_positive_addr!r}" + + +def test_street_regex_param_passed_to_db(trade_in_app: FastAPI) -> None: + """Endpoint passes street_regex with word-boundary markers to DB execute().""" + db_mock = _make_db_mock([]) + + from app.core.db import get_db + + def _override(): + yield db_mock + + trade_in_app.dependency_overrides[get_db] = _override + + client = TestClient(trade_in_app) + client.get( + "/api/v1/trade-in/street-deals", + params={ + "address": "г. Екатеринбург, ул. Мира, 10", + "area_m2": 50.0, + "rooms": 2, + }, + ) + + assert db_mock.execute.called, "db.execute should have been called" + call_args = db_mock.execute.call_args + params = call_args[0][1] if len(call_args[0]) > 1 else call_args[1].get("parameters", {}) + assert "street_regex" in params, f"street_regex not in params: {params}" + regex_val = params["street_regex"] + # Must contain word-boundary anchors + assert ( + r"\m" in regex_val or r"\b" in regex_val or regex_val.startswith(r"\m") + ), f"Expected word-boundary in regex, got: {regex_val!r}"