lekss361/gendesign
1
0
Fork 0
Code Issues 57 Pull requests Projects Releases Packages Wiki Activity Actions

fix(tradein): word-boundary street match + cian unescape robustness (Refs #769 A1+D) #798

Merged
bot-reviewer merged 1 commit from fix/769-street-match-cian-unescape into main 2026-05-30 17:18:34 +00:00
Conversation 0 Commits 1 Files changed 5 +123 -25
5 changed files with 123 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
tradein-mvp/backend/app/api/v1/trade_in.py
View file

@ -1478,6 +1478,7 @@ def get_street_deals(
FROM deals FROM deals
WHERE source = 'rosreestr' WHERE source = 'rosreestr'
AND address ILIKE :street_pattern AND address ILIKE :street_pattern
AND address ~* :street_regex
AND rooms = CAST(:rooms AS integer) AND rooms = CAST(:rooms AS integer)
AND area_m2 BETWEEN :area_min AND :area_max AND area_m2 BETWEEN :area_min AND :area_max
AND deal_date > NOW() - (CAST(:period_months AS integer) || ' months')::interval AND deal_date > NOW() - (CAST(:period_months AS integer) || ' months')::interval
@ -1487,6 +1488,7 @@ def get_street_deals(
), ),
{ {
"street_pattern": "%" + street_name + "%", "street_pattern": "%" + street_name + "%",
"street_regex": r"\m" + street_name + r"\M",
"rooms": rooms, "rooms": rooms,
"area_min": area_min, "area_min": area_min,
"area_max": area_max, "area_max": area_max,

2
tradein-mvp/backend/app/services/estimator.py
View file

@ -821,6 +821,7 @@ def _fetch_dkp_corridor(
FROM deals FROM deals
WHERE source = 'rosreestr' WHERE source = 'rosreestr'
AND address ILIKE :street_pattern AND address ILIKE :street_pattern
AND address ~* :street_regex
AND rooms = CAST(:rooms AS integer) AND rooms = CAST(:rooms AS integer)
AND area_m2 BETWEEN :area_min AND :area_max AND area_m2 BETWEEN :area_min AND :area_max
AND deal_date > NOW() AND deal_date > NOW()
@ -832,6 +833,7 @@ def _fetch_dkp_corridor(
), ),
{ {
"street_pattern": "%" + street_name + "%", "street_pattern": "%" + street_name + "%",
"street_regex": r"\m" + street_name + r"\M",
"rooms": rooms, "rooms": rooms,
"area_min": area_min, "area_min": area_min,
"area_max": area_max, "area_max": area_max,

24
tradein-mvp/backend/app/services/scrapers/cian_state_parser.py
View file

@ -3,6 +3,7 @@
Pattern: window._cianConfig['<mfe>'].push({key:..., value:..., priority:..., filter:...}) Pattern: window._cianConfig['<mfe>'].push({key:..., value:..., priority:..., filter:...})
Used by SERP / detail / newbuilding / valuation scrapers. Used by SERP / detail / newbuilding / valuation scrapers.
""" """
import json import json
import logging import logging
import re import re
@ -45,23 +46,24 @@ def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, A
pass pass
# Strategy 2: JSON.parse("escaped string") # Strategy 2: JSON.parse("escaped string")
m_jp = re.match(r'JSON\.parse\(\s*"(.+?)"\s*\)\s*$', value_str, re.DOTALL) # Use json.loads on the outer quoted string to unescape atomically — avoids
# order-dependent manual .replace() chains that break when descriptions contain
# backslashes (e.g. '\\' in listing text corrupts the rest of the parse).
m_jp = re.match(r'JSON\.parse\(\s*("(?:[^"\\]|\\.)*")\s*\)\s*$', value_str, re.DOTALL)
if m_jp: if m_jp:
escaped = m_jp.group(1) outer_json_str = m_jp.group(1) # the full "..." including surrounding quotes
unescaped = (
escaped
.replace('\\"', '"')
.replace('\\\\', '\\')
.replace('\\n', '\n')
.replace('\\/', '/')
)
try: try:
return json.loads(unescaped) inner = json.loads(outer_json_str) # inner is now a Python str
except json.JSONDecodeError as exc: except json.JSONDecodeError as exc:
logger.debug( logger.debug(
"JSON.parse() unescape failed for mfe=%s key=%s: %s", mfe, key, exc "JSON.parse() outer unescape failed for mfe=%s key=%s: %s", mfe, key, exc
) )
continue continue
try:
return json.loads(inner)
except json.JSONDecodeError as exc:
logger.debug("JSON.parse() inner parse failed for mfe=%s key=%s: %s", mfe, key, exc)
continue
# Strategy 3: raw JS object literal — try demjson3 (optional dep) # Strategy 3: raw JS object literal — try demjson3 (optional dep)
try: try:

58
tradein-mvp/backend/tests/test_cian_state_parser.py
View file

@ -1,10 +1,10 @@
"""Tests for cian_state_parser.""" """Tests for cian_state_parser."""
import pytest
from app.services.scrapers.cian_state_parser import extract_state from app.services.scrapers.cian_state_parser import extract_state
def test_extract_state_simple_json(): def test_extract_state_simple_json():
html = ''' html = """
<html><body> <html><body>
<script> <script>
window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || []; window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];
@ -15,32 +15,64 @@ def test_extract_state_simple_json():
}); });
</script> </script>
</body></html> </body></html>
''' """
state = extract_state(html, mfe='frontend-serp', key='initialState') state = extract_state(html, mfe="frontend-serp", key="initialState")
assert state is not None assert state is not None
assert state['results']['offers'][0]['id'] == 1 assert state["results"]["offers"][0]["id"] == 1
def test_extract_state_json_parse_form(): def test_extract_state_json_parse_form():
html = r''' html = r"""
window._cianConfig['frontend-offer-card'].push({ window._cianConfig['frontend-offer-card'].push({
key: 'defaultState', key: 'defaultState',
value: JSON.parse("{\"offerData\":{\"offer\":{\"cianId\":123}}}"), value: JSON.parse("{\"offerData\":{\"offer\":{\"cianId\":123}}}"),
priority: 1 priority: 1
}); });
''' """
state = extract_state(html, mfe='frontend-offer-card', key='defaultState') state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
assert state is not None assert state is not None
assert state['offerData']['offer']['cianId'] == 123 assert state["offerData"]["offer"]["cianId"] == 123
def test_extract_state_missing_returns_none(): def test_extract_state_missing_returns_none():
html = "<html></html>" html = "<html></html>"
assert extract_state(html, mfe='frontend-serp') is None assert extract_state(html, mfe="frontend-serp") is None
def test_extract_state_wrong_mfe_returns_none(): def test_extract_state_wrong_mfe_returns_none():
html = ''' html = """
window._cianConfig['other-mfe'].push({key: 'initialState', value: {}, priority: 1}); window._cianConfig['other-mfe'].push({key: 'initialState', value: {}, priority: 1});
''' """
assert extract_state(html, mfe='frontend-serp', key='initialState') is None assert extract_state(html, mfe="frontend-serp", key="initialState") is None
def test_extract_state_backslash_in_description() -> None:
"""Backslash in a listing description must not break page parsing (Part D fix).
The hand-rolled .replace() chain was order-dependent: '\\\\' → '\\' after
'\\"' → '"' left stray backslashes that broke json.loads on the inner payload.
The fix uses json.loads on the outer quoted string atomically.
"""
# Build a JSON payload where the description contains a literal backslash.
# In the wire format (JSON.parse("...")), the inner JSON is double-escaped:
# inner JSON: {"desc": "с \\ слешем"}
# outer string: "{\"desc\": \"с \\\\ слешем\"}"
import json
inner_payload = {"offerData": {"offer": {"cianId": 42, "description": r"описание с \ слешем"}}}
inner_json_str = json.dumps(inner_payload, ensure_ascii=False)
# Simulate what Cian emits: JSON.parse("<double-escaped inner>")
outer_escaped = json.dumps(inner_json_str) # produces "\"...\"" with double escaping
html = f"""
window._cianConfig['frontend-offer-card'].push({{
key: 'defaultState',
value: JSON.parse({outer_escaped}),
priority: 1
}});
"""
state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
assert state is not None, "extract_state returned None — page parse failed"
assert state["offerData"]["offer"]["cianId"] == 42
assert (
"\\" in state["offerData"]["offer"]["description"]
), "Backslash in description was lost during unescape"

62
tradein-mvp/backend/tests/test_street_deals_endpoint.py
View file

@ -233,10 +233,70 @@ def test_street_deals_aggregates_correctly(trade_in_app: FastAPI) -> None:
assert data["street"] == "Космонавтов" assert data["street"] == "Космонавтов"
assert data["count"] == 5 assert data["count"] == 5
assert data["median_price_per_m2"] == 100_000 assert data["median_price_per_m2"] == 100_000
assert data["median_price_rub"] == 5_000_000 # 100k * 50m² assert data["median_price_rub"] == 5_000_000 # 100k * 50m²
assert data["range_low_rub"] == 4_000_000 assert data["range_low_rub"] == 4_000_000
assert data["range_high_rub"] == 6_000_000 assert data["range_high_rub"] == 6_000_000
# All 5 deals returned (top-10 threshold) # All 5 deals returned (top-10 threshold)
assert len(data["deals"]) == 5 assert len(data["deals"]) == 5
# DB rows already ordered by deal_date DESC — first is most recent (2025-10-01) # DB rows already ordered by deal_date DESC — first is most recent (2025-10-01)
assert data["deals"][0]["price_rub"] == 6_000_000 assert data["deals"][0]["price_rub"] == 6_000_000
# ── Test: word-boundary street match (Part A1 fix) ───────────────────────────
def test_street_regex_word_boundary_no_false_positive() -> None:
"""«Мира» must NOT match «улица Адмирала Макарова» (contains 'мира' as substring).
We verify the pattern construction directly using Python re (\\b is equivalent
to Postgres \\m/\\M for word-boundary checks on these inputs).
"""
import re
street = "Мира"
pattern = r"\b" + re.escape(street) + r"\b"
# Must NOT match — 'мира' appears inside 'Макарова' substring check skipped
# but more critically — 'мира' appears inside 'Адмирала Макарова'
false_positive_addr = "улица Адмирала Макарова"
assert not re.search(
pattern, false_positive_addr, re.IGNORECASE
), f"Pattern {pattern!r} should NOT match {false_positive_addr!r}"
# Must match — exact word
true_positive_addr = "улица Мира 5"
assert re.search(
pattern, true_positive_addr, re.IGNORECASE
), f"Pattern {pattern!r} should match {true_positive_addr!r}"
def test_street_regex_param_passed_to_db(trade_in_app: FastAPI) -> None:
"""Endpoint passes street_regex with word-boundary markers to DB execute()."""
db_mock = _make_db_mock([])
from app.core.db import get_db
def _override():
yield db_mock
trade_in_app.dependency_overrides[get_db] = _override
client = TestClient(trade_in_app)
client.get(
"/api/v1/trade-in/street-deals",
params={
"address": "г. Екатеринбург, ул. Мира, 10",
"area_m2": 50.0,
"rooms": 2,
},
)
assert db_mock.execute.called, "db.execute should have been called"
call_args = db_mock.execute.call_args
params = call_args[0][1] if len(call_args[0]) > 1 else call_args[1].get("parameters", {})
assert "street_regex" in params, f"street_regex not in params: {params}"
regex_val = params["street_regex"]
# Must contain word-boundary anchors
assert (
r"\m" in regex_val or r"\b" in regex_val or regex_val.startswith(r"\m")
), f"Expected word-boundary in regex, got: {regex_val!r}"