"""Shared utility для извлечения Cian Redux state из _cianConfig. Pattern: window._cianConfig[''].push({key:..., value:..., priority:..., filter:...}) Used by SERP / detail / newbuilding / valuation scrapers. """ import json import logging import re from typing import Any logger = logging.getLogger(__name__) # Match both patterns Cian uses: # 1. window._cianConfig['mfe'].push({key:..., value:..., priority:..., filter:...}) # 2. window._cianConfig['mfe'] = window._cianConfig['mfe'] || []; # window._cianConfig['mfe'].push({...}) _RE_CIAN_PUSH = re.compile( r"window\._cianConfig\['(?P[\w-]+)'\]" r"(?:\.push\(\s*|" r"\s*=\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\];\s*" r"window\._cianConfig\['[\w-]+'\]\.push\(\s*)" r"\{\s*key:\s*['\"](?P\w+)['\"]\s*,\s*" r"value:\s*(?P.+?)\s*,\s*" r"priority:", re.DOTALL, ) # 2026-05 drift (#639): Cian сменил формат на # window._cianConfig['mfe'] = (window._cianConfig['mfe'] || []).concat([{"key":..,"value":..},..]) # вместо .push({key:..}). Ключи теперь в JSON-кавычках, аргумент concat([...]) — валидный # JSON-массив → парсим его целиком (надёжнее regexp по одному value). Старый .push формат # (_RE_CIAN_PUSH) оставлен как fallback для legacy-фикстур. _RE_CIAN_CONCAT = re.compile( r"window\._cianConfig\['(?P[\w-]+)'\]\s*=\s*" r"\(\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\s*\)\.concat\(\s*" ) def _balanced_array(s: str, start: int) -> str | None: """Срез s[start:end] — сбалансированный ``[...]`` от индекса '[', с учётом JSON-строк.""" depth = 0 in_str = False esc = False for i in range(start, len(s)): c = s[i] if in_str: if esc: esc = False elif c == "\\": esc = True elif c == '"': in_str = False continue if c == '"': in_str = True elif c == "[": depth += 1 elif c == "]": depth -= 1 if depth == 0: return s[start : i + 1] return None def _extract_from_concat(html: str, mfe: str, key: str) -> Any | None: """Новый формат: .concat([{"key":..,"value":..},..]) — массив-аргумент = валидный JSON.""" for m in _RE_CIAN_CONCAT.finditer(html): if m.group("mfe") != mfe: continue start = m.end() if start >= len(html) or html[start] != "[": start = html.find("[", m.end()) if start < 0: continue arr = _balanced_array(html, start) if arr is None: continue try: items = json.loads(arr) except json.JSONDecodeError as exc: logger.debug("concat array parse failed for mfe=%s key=%s: %s", mfe, key, exc) continue for it in items: if isinstance(it, dict) and it.get("key") == key: return it.get("value") return None def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, Any] | None: """Extract state.value where _cianConfig[] entry.key matches. Поддерживает оба формата Cian: новый ``.concat([{"key":..,"value":..}])`` (2026-05, #639) и legacy ``.push({key:..,value:..})``. Returns parsed JSON dict, or None. """ # Новый формат (concat) — приоритетный. concat_val = _extract_from_concat(html, mfe, key) if concat_val is not None: return concat_val for match in _RE_CIAN_PUSH.finditer(html): if match.group("mfe") != mfe or match.group("key") != key: continue value_str = match.group("value").strip() # Strategy 1: direct JSON parse (value уже JSON-like) try: return json.loads(value_str) except json.JSONDecodeError: pass # Strategy 2: JSON.parse("escaped string") # Use json.loads on the outer quoted string to unescape atomically — avoids # order-dependent manual .replace() chains that break when descriptions contain # backslashes (e.g. '\\' in listing text corrupts the rest of the parse). m_jp = re.match(r'JSON\.parse\(\s*("(?:[^"\\]|\\.)*")\s*\)\s*$', value_str, re.DOTALL) if m_jp: outer_json_str = m_jp.group(1) # the full "..." including surrounding quotes try: inner = json.loads(outer_json_str) # inner is now a Python str except json.JSONDecodeError as exc: logger.debug( "JSON.parse() outer unescape failed for mfe=%s key=%s: %s", mfe, key, exc ) continue try: return json.loads(inner) except json.JSONDecodeError as exc: logger.debug("JSON.parse() inner parse failed for mfe=%s key=%s: %s", mfe, key, exc) continue # Strategy 3: raw JS object literal — try demjson3 (optional dep) try: import demjson3 # type: ignore[import-untyped] return demjson3.decode(value_str) except ImportError: logger.debug( "demjson3 not installed; cannot parse raw JS object for mfe=%s key=%s", mfe, key ) except Exception as exc: logger.debug("demjson3 parse failed for mfe=%s key=%s: %s", mfe, key, exc) return None def _extract_all_from_concat(html: str) -> dict[str, dict[str, Any]]: """Новый формат (``.concat([...])``) — discovery-версия `_extract_from_concat` без key-фильтра: собирает ВСЕ key/value пары из каждого concat-массива сразу (один concat-вызов на mfe несёт сразу несколько key/value записей).""" result: dict[str, dict[str, Any]] = {} for m in _RE_CIAN_CONCAT.finditer(html): mfe = m.group("mfe") start = m.end() if start >= len(html) or html[start] != "[": start = html.find("[", m.end()) if start < 0: continue arr = _balanced_array(html, start) if arr is None: continue try: items = json.loads(arr) except json.JSONDecodeError as exc: logger.debug("concat array parse failed for mfe=%s (discovery): %s", mfe, exc) continue for it in items: if isinstance(it, dict) and "key" in it: result.setdefault(mfe, {})[it["key"]] = it.get("value") return result def extract_all_states(html: str) -> dict[str, dict[str, Any]]: """Returns {mfe_name: {key: parsed_value}} for all push()/concat() entries. Useful for discovery / debugging during scraper development. Bug fix (found live 2026-07-04 while verifying #2435 bti_data persistence): this previously only scanned the legacy `.push({...})` pattern (`_RE_CIAN_PUSH`) — but Cian switched to `.concat([...])` in 2026-05 (#639), which `extract_state()` already handles (concat checked first, push as fallback) while this function never did. Result: on current-format pages this returned `{}` unconditionally, silently starving every `extract_all_states()` consumer (bti sister-state in detail.py, newbuilding sister-state in newbuilding.py) of real data. Mirrors extract_state()'s concat-first / push-fallback precedence. """ result = _extract_all_from_concat(html) for match in _RE_CIAN_PUSH.finditer(html): mfe = match.group("mfe") key = match.group("key") if key in result.get(mfe, {}): continue # concat already supplied this mfe/key — don't reparse/overwrite parsed = extract_state(html, mfe, key) if parsed is not None: result.setdefault(mfe, {})[key] = parsed return result