fix(tradein): word-boundary street match + cian unescape robustness (Refs #769 A1+D)
Часть A1 (#10 correctness): address ILIKE '%street%' давал ложные совпадения («Мира» матчит «Адмирала Макарова») → раздувало аналоги оценки. Добавлен word-boundary regex (~* :street_regex, \m..\M) рядом с ILIKE-prefilter в trade_in.py (street-deals) и estimator.py (_fetch_dkp_corridor). Bind-param, не f-string. Только предикат street-match, anchor/pricing не тронуты. Часть D (#16): cian_state_parser hand-rolled unescape был order-dependent (\" заменялся раньше \ → порча) → backslash в описании ронял парс всей страницы. Заменён на атомарный json.loads внешней quoted-строки. Тесты: «Мира»≠«Адмирала Макарова» но =«Мира 5»; backslash-в-описании парсится. 22 passed, ruff clean. A2 (trgm index) / B (FDW options) / C (matview refresh) / E (geo centroid) ОТЛОЖЕНЫ — нужна live tradein-DB / investigation (follow-up в #769).
This commit is contained in:
parent
a4057a5b28
commit
10155eb0fb
5 changed files with 123 additions and 25 deletions
|
|
@ -1478,6 +1478,7 @@ def get_street_deals(
|
|||
FROM deals
|
||||
WHERE source = 'rosreestr'
|
||||
AND address ILIKE :street_pattern
|
||||
AND address ~* :street_regex
|
||||
AND rooms = CAST(:rooms AS integer)
|
||||
AND area_m2 BETWEEN :area_min AND :area_max
|
||||
AND deal_date > NOW() - (CAST(:period_months AS integer) || ' months')::interval
|
||||
|
|
@ -1487,6 +1488,7 @@ def get_street_deals(
|
|||
),
|
||||
{
|
||||
"street_pattern": "%" + street_name + "%",
|
||||
"street_regex": r"\m" + street_name + r"\M",
|
||||
"rooms": rooms,
|
||||
"area_min": area_min,
|
||||
"area_max": area_max,
|
||||
|
|
|
|||
|
|
@ -821,6 +821,7 @@ def _fetch_dkp_corridor(
|
|||
FROM deals
|
||||
WHERE source = 'rosreestr'
|
||||
AND address ILIKE :street_pattern
|
||||
AND address ~* :street_regex
|
||||
AND rooms = CAST(:rooms AS integer)
|
||||
AND area_m2 BETWEEN :area_min AND :area_max
|
||||
AND deal_date > NOW()
|
||||
|
|
@ -832,6 +833,7 @@ def _fetch_dkp_corridor(
|
|||
),
|
||||
{
|
||||
"street_pattern": "%" + street_name + "%",
|
||||
"street_regex": r"\m" + street_name + r"\M",
|
||||
"rooms": rooms,
|
||||
"area_min": area_min,
|
||||
"area_max": area_max,
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
Pattern: window._cianConfig['<mfe>'].push({key:..., value:..., priority:..., filter:...})
|
||||
Used by SERP / detail / newbuilding / valuation scrapers.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
|
@ -45,23 +46,24 @@ def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, A
|
|||
pass
|
||||
|
||||
# Strategy 2: JSON.parse("escaped string")
|
||||
m_jp = re.match(r'JSON\.parse\(\s*"(.+?)"\s*\)\s*$', value_str, re.DOTALL)
|
||||
# Use json.loads on the outer quoted string to unescape atomically — avoids
|
||||
# order-dependent manual .replace() chains that break when descriptions contain
|
||||
# backslashes (e.g. '\\' in listing text corrupts the rest of the parse).
|
||||
m_jp = re.match(r'JSON\.parse\(\s*("(?:[^"\\]|\\.)*")\s*\)\s*$', value_str, re.DOTALL)
|
||||
if m_jp:
|
||||
escaped = m_jp.group(1)
|
||||
unescaped = (
|
||||
escaped
|
||||
.replace('\\"', '"')
|
||||
.replace('\\\\', '\\')
|
||||
.replace('\\n', '\n')
|
||||
.replace('\\/', '/')
|
||||
)
|
||||
outer_json_str = m_jp.group(1) # the full "..." including surrounding quotes
|
||||
try:
|
||||
return json.loads(unescaped)
|
||||
inner = json.loads(outer_json_str) # inner is now a Python str
|
||||
except json.JSONDecodeError as exc:
|
||||
logger.debug(
|
||||
"JSON.parse() unescape failed for mfe=%s key=%s: %s", mfe, key, exc
|
||||
"JSON.parse() outer unescape failed for mfe=%s key=%s: %s", mfe, key, exc
|
||||
)
|
||||
continue
|
||||
try:
|
||||
return json.loads(inner)
|
||||
except json.JSONDecodeError as exc:
|
||||
logger.debug("JSON.parse() inner parse failed for mfe=%s key=%s: %s", mfe, key, exc)
|
||||
continue
|
||||
|
||||
# Strategy 3: raw JS object literal — try demjson3 (optional dep)
|
||||
try:
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
"""Tests for cian_state_parser."""
|
||||
import pytest
|
||||
|
||||
from app.services.scrapers.cian_state_parser import extract_state
|
||||
|
||||
|
||||
def test_extract_state_simple_json():
|
||||
html = '''
|
||||
html = """
|
||||
<html><body>
|
||||
<script>
|
||||
window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];
|
||||
|
|
@ -15,32 +15,64 @@ def test_extract_state_simple_json():
|
|||
});
|
||||
</script>
|
||||
</body></html>
|
||||
'''
|
||||
state = extract_state(html, mfe='frontend-serp', key='initialState')
|
||||
"""
|
||||
state = extract_state(html, mfe="frontend-serp", key="initialState")
|
||||
assert state is not None
|
||||
assert state['results']['offers'][0]['id'] == 1
|
||||
assert state["results"]["offers"][0]["id"] == 1
|
||||
|
||||
|
||||
def test_extract_state_json_parse_form():
|
||||
html = r'''
|
||||
html = r"""
|
||||
window._cianConfig['frontend-offer-card'].push({
|
||||
key: 'defaultState',
|
||||
value: JSON.parse("{\"offerData\":{\"offer\":{\"cianId\":123}}}"),
|
||||
priority: 1
|
||||
});
|
||||
'''
|
||||
state = extract_state(html, mfe='frontend-offer-card', key='defaultState')
|
||||
"""
|
||||
state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
|
||||
assert state is not None
|
||||
assert state['offerData']['offer']['cianId'] == 123
|
||||
assert state["offerData"]["offer"]["cianId"] == 123
|
||||
|
||||
|
||||
def test_extract_state_missing_returns_none():
|
||||
html = "<html></html>"
|
||||
assert extract_state(html, mfe='frontend-serp') is None
|
||||
assert extract_state(html, mfe="frontend-serp") is None
|
||||
|
||||
|
||||
def test_extract_state_wrong_mfe_returns_none():
|
||||
html = '''
|
||||
html = """
|
||||
window._cianConfig['other-mfe'].push({key: 'initialState', value: {}, priority: 1});
|
||||
'''
|
||||
assert extract_state(html, mfe='frontend-serp', key='initialState') is None
|
||||
"""
|
||||
assert extract_state(html, mfe="frontend-serp", key="initialState") is None
|
||||
|
||||
|
||||
def test_extract_state_backslash_in_description() -> None:
|
||||
"""Backslash in a listing description must not break page parsing (Part D fix).
|
||||
|
||||
The hand-rolled .replace() chain was order-dependent: '\\\\' → '\\' after
|
||||
'\\"' → '"' left stray backslashes that broke json.loads on the inner payload.
|
||||
The fix uses json.loads on the outer quoted string atomically.
|
||||
"""
|
||||
# Build a JSON payload where the description contains a literal backslash.
|
||||
# In the wire format (JSON.parse("...")), the inner JSON is double-escaped:
|
||||
# inner JSON: {"desc": "с \\ слешем"}
|
||||
# outer string: "{\"desc\": \"с \\\\ слешем\"}"
|
||||
import json
|
||||
|
||||
inner_payload = {"offerData": {"offer": {"cianId": 42, "description": r"описание с \ слешем"}}}
|
||||
inner_json_str = json.dumps(inner_payload, ensure_ascii=False)
|
||||
# Simulate what Cian emits: JSON.parse("<double-escaped inner>")
|
||||
outer_escaped = json.dumps(inner_json_str) # produces "\"...\"" with double escaping
|
||||
html = f"""
|
||||
window._cianConfig['frontend-offer-card'].push({{
|
||||
key: 'defaultState',
|
||||
value: JSON.parse({outer_escaped}),
|
||||
priority: 1
|
||||
}});
|
||||
"""
|
||||
state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
|
||||
assert state is not None, "extract_state returned None — page parse failed"
|
||||
assert state["offerData"]["offer"]["cianId"] == 42
|
||||
assert (
|
||||
"\\" in state["offerData"]["offer"]["description"]
|
||||
), "Backslash in description was lost during unescape"
|
||||
|
|
|
|||
|
|
@ -233,10 +233,70 @@ def test_street_deals_aggregates_correctly(trade_in_app: FastAPI) -> None:
|
|||
assert data["street"] == "Космонавтов"
|
||||
assert data["count"] == 5
|
||||
assert data["median_price_per_m2"] == 100_000
|
||||
assert data["median_price_rub"] == 5_000_000 # 100k * 50m²
|
||||
assert data["median_price_rub"] == 5_000_000 # 100k * 50m²
|
||||
assert data["range_low_rub"] == 4_000_000
|
||||
assert data["range_high_rub"] == 6_000_000
|
||||
# All 5 deals returned (top-10 threshold)
|
||||
assert len(data["deals"]) == 5
|
||||
# DB rows already ordered by deal_date DESC — first is most recent (2025-10-01)
|
||||
assert data["deals"][0]["price_rub"] == 6_000_000
|
||||
|
||||
|
||||
# ── Test: word-boundary street match (Part A1 fix) ───────────────────────────
|
||||
|
||||
|
||||
def test_street_regex_word_boundary_no_false_positive() -> None:
|
||||
"""«Мира» must NOT match «улица Адмирала Макарова» (contains 'мира' as substring).
|
||||
|
||||
We verify the pattern construction directly using Python re (\\b is equivalent
|
||||
to Postgres \\m/\\M for word-boundary checks on these inputs).
|
||||
"""
|
||||
import re
|
||||
|
||||
street = "Мира"
|
||||
pattern = r"\b" + re.escape(street) + r"\b"
|
||||
|
||||
# Must NOT match — 'мира' appears inside 'Макарова' substring check skipped
|
||||
# but more critically — 'мира' appears inside 'Адмирала Макарова'
|
||||
false_positive_addr = "улица Адмирала Макарова"
|
||||
assert not re.search(
|
||||
pattern, false_positive_addr, re.IGNORECASE
|
||||
), f"Pattern {pattern!r} should NOT match {false_positive_addr!r}"
|
||||
|
||||
# Must match — exact word
|
||||
true_positive_addr = "улица Мира 5"
|
||||
assert re.search(
|
||||
pattern, true_positive_addr, re.IGNORECASE
|
||||
), f"Pattern {pattern!r} should match {true_positive_addr!r}"
|
||||
|
||||
|
||||
def test_street_regex_param_passed_to_db(trade_in_app: FastAPI) -> None:
|
||||
"""Endpoint passes street_regex with word-boundary markers to DB execute()."""
|
||||
db_mock = _make_db_mock([])
|
||||
|
||||
from app.core.db import get_db
|
||||
|
||||
def _override():
|
||||
yield db_mock
|
||||
|
||||
trade_in_app.dependency_overrides[get_db] = _override
|
||||
|
||||
client = TestClient(trade_in_app)
|
||||
client.get(
|
||||
"/api/v1/trade-in/street-deals",
|
||||
params={
|
||||
"address": "г. Екатеринбург, ул. Мира, 10",
|
||||
"area_m2": 50.0,
|
||||
"rooms": 2,
|
||||
},
|
||||
)
|
||||
|
||||
assert db_mock.execute.called, "db.execute should have been called"
|
||||
call_args = db_mock.execute.call_args
|
||||
params = call_args[0][1] if len(call_args[0]) > 1 else call_args[1].get("parameters", {})
|
||||
assert "street_regex" in params, f"street_regex not in params: {params}"
|
||||
regex_val = params["street_regex"]
|
||||
# Must contain word-boundary anchors
|
||||
assert (
|
||||
r"\m" in regex_val or r"\b" in regex_val or regex_val.startswith(r"\m")
|
||||
), f"Expected word-boundary in regex, got: {regex_val!r}"
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue