65b7b73691
Cian сменил формат встраивания state (2026-05): `.push({key:..,value:..})` →
`window._cianConfig['mfe'] = (... || []).concat([{"key":..,"value":..},..])` с
JSON-кавычками. `_RE_CIAN_PUSH` матчил 0 → extract_state=None → verify_session
заворачивал ВАЛИДНЫЕ куки → scheduler.py скипал cian-бэкфилл («cookies expired»).
Ломало весь cian (SERP/valuation/detail/newbuilding — общий extract_state).
Плюс auth-сигнал (user.isAuthenticated/userId) уехал из valuation-MFE (там user
теперь anonymous) в site-header MFE `header-frontend`.
- cian_state_parser.py: `_extract_from_concat` (regex .concat([..]) + string-aware
balance-scan `[...]` → json.loads массива → item по key). extract_state пробует
concat первым, .push остаётся fallback для legacy-фикстур.
- cian_session.py: verify читает auth из `_MFE_AUTH='header-frontend'`, не valuation.
- tests: concat-формат, ']'/'[' внутри строк, legacy push fallback.
Диагностировано вживую на проде (валидная сессия userId 75544897): _RE_CIAN_PUSH=0
матчей при 12 живых _cianConfig; concat-парсер извлекает initialState; isAuth=true
в header-frontend. 34 passed, ruff clean.
Refs #639, #806.
122 lines
4.8 KiB
Python
122 lines
4.8 KiB
Python
"""Tests for cian_state_parser."""
|
||
|
||
from app.services.scrapers.cian_state_parser import extract_state
|
||
|
||
|
||
def test_extract_state_simple_json():
|
||
html = """
|
||
<html><body>
|
||
<script>
|
||
window._cianConfig['frontend-serp'] = window._cianConfig['frontend-serp'] || [];
|
||
window._cianConfig['frontend-serp'].push({
|
||
key: 'initialState',
|
||
value: {"results": {"offers": [{"id": 1}]}},
|
||
priority: 1
|
||
});
|
||
</script>
|
||
</body></html>
|
||
"""
|
||
state = extract_state(html, mfe="frontend-serp", key="initialState")
|
||
assert state is not None
|
||
assert state["results"]["offers"][0]["id"] == 1
|
||
|
||
|
||
def test_extract_state_json_parse_form():
|
||
html = r"""
|
||
window._cianConfig['frontend-offer-card'].push({
|
||
key: 'defaultState',
|
||
value: JSON.parse("{\"offerData\":{\"offer\":{\"cianId\":123}}}"),
|
||
priority: 1
|
||
});
|
||
"""
|
||
state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
|
||
assert state is not None
|
||
assert state["offerData"]["offer"]["cianId"] == 123
|
||
|
||
|
||
def test_extract_state_missing_returns_none():
|
||
html = "<html></html>"
|
||
assert extract_state(html, mfe="frontend-serp") is None
|
||
|
||
|
||
def test_extract_state_wrong_mfe_returns_none():
|
||
html = """
|
||
window._cianConfig['other-mfe'].push({key: 'initialState', value: {}, priority: 1});
|
||
"""
|
||
assert extract_state(html, mfe="frontend-serp", key="initialState") is None
|
||
|
||
|
||
def test_extract_state_backslash_in_description() -> None:
|
||
"""Backslash in a listing description must not break page parsing (Part D fix).
|
||
|
||
The hand-rolled .replace() chain was order-dependent: '\\\\' → '\\' after
|
||
'\\"' → '"' left stray backslashes that broke json.loads on the inner payload.
|
||
The fix uses json.loads on the outer quoted string atomically.
|
||
"""
|
||
# Build a JSON payload where the description contains a literal backslash.
|
||
# In the wire format (JSON.parse("...")), the inner JSON is double-escaped:
|
||
# inner JSON: {"desc": "с \\ слешем"}
|
||
# outer string: "{\"desc\": \"с \\\\ слешем\"}"
|
||
import json
|
||
|
||
inner_payload = {"offerData": {"offer": {"cianId": 42, "description": r"описание с \ слешем"}}}
|
||
inner_json_str = json.dumps(inner_payload, ensure_ascii=False)
|
||
# Simulate what Cian emits: JSON.parse("<double-escaped inner>")
|
||
outer_escaped = json.dumps(inner_json_str) # produces "\"...\"" with double escaping
|
||
html = f"""
|
||
window._cianConfig['frontend-offer-card'].push({{
|
||
key: 'defaultState',
|
||
value: JSON.parse({outer_escaped}),
|
||
priority: 1
|
||
}});
|
||
"""
|
||
state = extract_state(html, mfe="frontend-offer-card", key="defaultState")
|
||
assert state is not None, "extract_state returned None — page parse failed"
|
||
assert state["offerData"]["offer"]["cianId"] == 42
|
||
assert (
|
||
"\\" in state["offerData"]["offer"]["description"]
|
||
), "Backslash in description was lost during unescape"
|
||
|
||
|
||
# ── #639: новый формат Cian (2026-05) — .concat([{"key":..,"value":..}]) ──
|
||
|
||
|
||
def test_extract_state_concat_format() -> None:
|
||
"""Новый формат: .concat([{...}]) с JSON-кавычками вместо .push({key:..})."""
|
||
html = (
|
||
"<script>window._cianConfig = window._cianConfig || {};"
|
||
"window._cianConfig['frontend-serp'] = "
|
||
"(window._cianConfig['frontend-serp'] || []).concat("
|
||
'[{"key":"projectName","value":"frontend-serp","priority":1000,"filter":{}},'
|
||
'{"key":"initialState","value":{"results":{"offers":[{"id":7}]}},"priority":1,"filter":{}}]'
|
||
");</script>"
|
||
)
|
||
state = extract_state(html, mfe="frontend-serp", key="initialState")
|
||
assert state is not None
|
||
assert state["results"]["offers"][0]["id"] == 7
|
||
|
||
|
||
def test_extract_state_concat_bracket_in_string() -> None:
|
||
"""Балансировщик массива не ломается на ']'/'[' внутри JSON-строк."""
|
||
html = (
|
||
"window._cianConfig['header-frontend'] = "
|
||
"(window._cianConfig['header-frontend'] || []).concat("
|
||
'[{"key":"initialState","value":{"user":{"isAuthenticated":true,"userId":42,'
|
||
'"note":"text with ] and [ inside"}},"priority":1}]'
|
||
");"
|
||
)
|
||
state = extract_state(html, mfe="header-frontend", key="initialState")
|
||
assert state is not None
|
||
assert state["user"]["isAuthenticated"] is True
|
||
assert state["user"]["userId"] == 42
|
||
|
||
|
||
def test_extract_state_legacy_push_still_works() -> None:
|
||
"""Старый .push формат остаётся рабочим (fallback)."""
|
||
html = (
|
||
"window._cianConfig['frontend-serp'].push({"
|
||
'key: \'initialState\', value: {"results":{"offers":[{"id":3}]}}, priority: 1});'
|
||
)
|
||
state = extract_state(html, mfe="frontend-serp", key="initialState")
|
||
assert state is not None
|
||
assert state["results"]["offers"][0]["id"] == 3
|