From 7bd5a834dc204e71aca847ca02c8d21ec93dfd9b Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 30 May 2026 19:08:01 +0000 Subject: [PATCH] =?UTF-8?q?fix(tradein):=20cian=20=5FcianConfig=20concat-?= =?UTF-8?q?=D1=84=D0=BE=D1=80=D0=BC=D0=B0=D1=82=20+=20auth=20=D0=B8=D0=B7?= =?UTF-8?q?=20header-frontend=20(#639)=20(#819)=20Co-authored-by:=20bot-ba?= =?UTF-8?q?ckend=20=20Co-committed-by:=20bot-ba?= =?UTF-8?q?ckend=20?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/app/services/cian_session.py | 6 +- .../services/scrapers/cian_state_parser.py | 69 ++++++++++++++++++- .../backend/tests/test_cian_state_parser.py | 44 ++++++++++++ 3 files changed, 116 insertions(+), 3 deletions(-) diff --git a/tradein-mvp/backend/app/services/cian_session.py b/tradein-mvp/backend/app/services/cian_session.py index ee45331d..8201bfbf 100644 --- a/tradein-mvp/backend/app/services/cian_session.py +++ b/tradein-mvp/backend/app/services/cian_session.py @@ -56,6 +56,10 @@ _VALUATION_TEST_URL = ( ) _MFE_VALUATION = "valuation-for-agent-frontend" +# #639 (2026-05): auth-сигнал (user.isAuthenticated/userId) уехал из valuation-MFE +# (там user теперь всегда anonymous) в site-header MFE `header-frontend`, который +# присутствует на любой cian-странице. Verify читает auth именно оттуда. +_MFE_AUTH = "header-frontend" # Returned by verify_session when a TLS/bot-fingerprint ban (HTTP 403) is detected. # Callers that check `if state is None` will NOT treat this as "expired cookies" — @@ -81,7 +85,7 @@ def _classify_verify_response( return None if html is None: return None - state = extract_state(html, mfe=_MFE_VALUATION, key="initialState") + state = extract_state(html, mfe=_MFE_AUTH, key="initialState") if state is None: return None user = state.get("user", {}) or {} diff --git a/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py b/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py index f0fef80c..f6a46a7a 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_state_parser.py @@ -26,13 +26,78 @@ _RE_CIAN_PUSH = re.compile( re.DOTALL, ) +# 2026-05 drift (#639): Cian сменил формат на +# window._cianConfig['mfe'] = (window._cianConfig['mfe'] || []).concat([{"key":..,"value":..},..]) +# вместо .push({key:..}). Ключи теперь в JSON-кавычках, аргумент concat([...]) — валидный +# JSON-массив → парсим его целиком (надёжнее regexp по одному value). Старый .push формат +# (_RE_CIAN_PUSH) оставлен как fallback для legacy-фикстур. +_RE_CIAN_CONCAT = re.compile( + r"window\._cianConfig\['(?P[\w-]+)'\]\s*=\s*" + r"\(\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\s*\)\.concat\(\s*" +) + + +def _balanced_array(s: str, start: int) -> str | None: + """Срез s[start:end] — сбалансированный ``[...]`` от индекса '[', с учётом JSON-строк.""" + depth = 0 + in_str = False + esc = False + for i in range(start, len(s)): + c = s[i] + if in_str: + if esc: + esc = False + elif c == "\\": + esc = True + elif c == '"': + in_str = False + continue + if c == '"': + in_str = True + elif c == "[": + depth += 1 + elif c == "]": + depth -= 1 + if depth == 0: + return s[start : i + 1] + return None + + +def _extract_from_concat(html: str, mfe: str, key: str) -> Any | None: + """Новый формат: .concat([{"key":..,"value":..},..]) — массив-аргумент = валидный JSON.""" + for m in _RE_CIAN_CONCAT.finditer(html): + if m.group("mfe") != mfe: + continue + start = m.end() + if start >= len(html) or html[start] != "[": + start = html.find("[", m.end()) + if start < 0: + continue + arr = _balanced_array(html, start) + if arr is None: + continue + try: + items = json.loads(arr) + except json.JSONDecodeError as exc: + logger.debug("concat array parse failed for mfe=%s key=%s: %s", mfe, key, exc) + continue + for it in items: + if isinstance(it, dict) and it.get("key") == key: + return it.get("value") + return None + def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, Any] | None: """Extract state.value where _cianConfig[] entry.key matches. - Returns parsed JSON dict, or None if not found / parse failed. - Logs warning at debug level on parse failure for diagnostics. + Поддерживает оба формата Cian: новый ``.concat([{"key":..,"value":..}])`` (2026-05, + #639) и legacy ``.push({key:..,value:..})``. Returns parsed JSON dict, or None. """ + # Новый формат (concat) — приоритетный. + concat_val = _extract_from_concat(html, mfe, key) + if concat_val is not None: + return concat_val + for match in _RE_CIAN_PUSH.finditer(html): if match.group("mfe") != mfe or match.group("key") != key: continue diff --git a/tradein-mvp/backend/tests/test_cian_state_parser.py b/tradein-mvp/backend/tests/test_cian_state_parser.py index 2c07035c..f1c15365 100644 --- a/tradein-mvp/backend/tests/test_cian_state_parser.py +++ b/tradein-mvp/backend/tests/test_cian_state_parser.py @@ -76,3 +76,47 @@ def test_extract_state_backslash_in_description() -> None: assert ( "\\" in state["offerData"]["offer"]["description"] ), "Backslash in description was lost during unescape" + + +# ── #639: новый формат Cian (2026-05) — .concat([{"key":..,"value":..}]) ── + + +def test_extract_state_concat_format() -> None: + """Новый формат: .concat([{...}]) с JSON-кавычками вместо .push({key:..}).""" + html = ( + "" + ) + state = extract_state(html, mfe="frontend-serp", key="initialState") + assert state is not None + assert state["results"]["offers"][0]["id"] == 7 + + +def test_extract_state_concat_bracket_in_string() -> None: + """Балансировщик массива не ломается на ']'/'[' внутри JSON-строк.""" + html = ( + "window._cianConfig['header-frontend'] = " + "(window._cianConfig['header-frontend'] || []).concat(" + '[{"key":"initialState","value":{"user":{"isAuthenticated":true,"userId":42,' + '"note":"text with ] and [ inside"}},"priority":1}]' + ");" + ) + state = extract_state(html, mfe="header-frontend", key="initialState") + assert state is not None + assert state["user"]["isAuthenticated"] is True + assert state["user"]["userId"] == 42 + + +def test_extract_state_legacy_push_still_works() -> None: + """Старый .push формат остаётся рабочим (fallback).""" + html = ( + "window._cianConfig['frontend-serp'].push({" + 'key: \'initialState\', value: {"results":{"offers":[{"id":3}]}}, priority: 1});' + ) + state = extract_state(html, mfe="frontend-serp", key="initialState") + assert state is not None + assert state["results"]["offers"][0]["id"] == 3