gendesign/tradein-mvp/packages/scraper-kit/src/scraper_kit/cian_state_parser.py

"""Shared utility для извлечения Cian Redux state из _cianConfig.

Pattern: window._cianConfig['<mfe>'].push({key:..., value:..., priority:..., filter:...})
Used by SERP / detail / newbuilding / valuation scrapers.
"""

import json
import logging
import re
from typing import Any

logger = logging.getLogger(__name__)

# Match both patterns Cian uses:
#   1. window._cianConfig['mfe'].push({key:..., value:..., priority:..., filter:...})
#   2. window._cianConfig['mfe'] = window._cianConfig['mfe'] || [];
#      window._cianConfig['mfe'].push({...})
_RE_CIAN_PUSH = re.compile(
    r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]"
    r"(?:\.push\(\s*|"
    r"\s*=\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\];\s*"
    r"window\._cianConfig\['[\w-]+'\]\.push\(\s*)"
    r"\{\s*key:\s*['\"](?P<key>\w+)['\"]\s*,\s*"
    r"value:\s*(?P<value>.+?)\s*,\s*"
    r"priority:",
    re.DOTALL,
)

# 2026-05 drift (#639): Cian сменил формат на
#   window._cianConfig['mfe'] = (window._cianConfig['mfe'] || []).concat([{"key":..,"value":..},..])
# вместо .push({key:..}). Ключи теперь в JSON-кавычках, аргумент concat([...]) — валидный
# JSON-массив → парсим его целиком (надёжнее regexp по одному value). Старый .push формат
# (_RE_CIAN_PUSH) оставлен как fallback для legacy-фикстур.
_RE_CIAN_CONCAT = re.compile(
    r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]\s*=\s*"
    r"\(\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\]\s*\)\.concat\(\s*"
)


def _balanced_array(s: str, start: int) -> str | None:
    """Срез s[start:end] — сбалансированный ``[...]`` от индекса '[', с учётом JSON-строк."""
    depth = 0
    in_str = False
    esc = False
    for i in range(start, len(s)):
        c = s[i]
        if in_str:
            if esc:
                esc = False
            elif c == "\\":
                esc = True
            elif c == '"':
                in_str = False
            continue
        if c == '"':
            in_str = True
        elif c == "[":
            depth += 1
        elif c == "]":
            depth -= 1
            if depth == 0:
                return s[start : i + 1]
    return None


def _extract_from_concat(html: str, mfe: str, key: str) -> Any | None:
    """Новый формат: .concat([{"key":..,"value":..},..]) — массив-аргумент = валидный JSON."""
    for m in _RE_CIAN_CONCAT.finditer(html):
        if m.group("mfe") != mfe:
            continue
        start = m.end()
        if start >= len(html) or html[start] != "[":
            start = html.find("[", m.end())
        if start < 0:
            continue
        arr = _balanced_array(html, start)
        if arr is None:
            continue
        try:
            items = json.loads(arr)
        except json.JSONDecodeError as exc:
            logger.debug("concat array parse failed for mfe=%s key=%s: %s", mfe, key, exc)
            continue
        for it in items:
            if isinstance(it, dict) and it.get("key") == key:
                return it.get("value")
    return None


def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, Any] | None:
    """Extract state.value where _cianConfig[<mfe>] entry.key matches.

    Поддерживает оба формата Cian: новый ``.concat([{"key":..,"value":..}])`` (2026-05,
    #639) и legacy ``.push({key:..,value:..})``. Returns parsed JSON dict, or None.
    """
    # Новый формат (concat) — приоритетный.
    concat_val = _extract_from_concat(html, mfe, key)
    if concat_val is not None:
        return concat_val

    for match in _RE_CIAN_PUSH.finditer(html):
        if match.group("mfe") != mfe or match.group("key") != key:
            continue

        value_str = match.group("value").strip()

        # Strategy 1: direct JSON parse (value уже JSON-like)
        try:
            return json.loads(value_str)
        except json.JSONDecodeError:
            pass

        # Strategy 2: JSON.parse("escaped string")
        # Use json.loads on the outer quoted string to unescape atomically — avoids
        # order-dependent manual .replace() chains that break when descriptions contain
        # backslashes (e.g. '\\' in listing text corrupts the rest of the parse).
        m_jp = re.match(r'JSON\.parse\(\s*("(?:[^"\\]|\\.)*")\s*\)\s*$', value_str, re.DOTALL)
        if m_jp:
            outer_json_str = m_jp.group(1)  # the full "..." including surrounding quotes
            try:
                inner = json.loads(outer_json_str)  # inner is now a Python str
            except json.JSONDecodeError as exc:
                logger.debug(
                    "JSON.parse() outer unescape failed for mfe=%s key=%s: %s", mfe, key, exc
                )
                continue
            try:
                return json.loads(inner)
            except json.JSONDecodeError as exc:
                logger.debug("JSON.parse() inner parse failed for mfe=%s key=%s: %s", mfe, key, exc)
                continue

        # Strategy 3: raw JS object literal — try demjson3 (optional dep)
        try:
            import demjson3  # type: ignore[import-untyped]

            return demjson3.decode(value_str)
        except ImportError:
            logger.debug(
                "demjson3 not installed; cannot parse raw JS object for mfe=%s key=%s", mfe, key
            )
        except Exception as exc:
            logger.debug("demjson3 parse failed for mfe=%s key=%s: %s", mfe, key, exc)

    return None


def _extract_all_from_concat(html: str) -> dict[str, dict[str, Any]]:
    """Новый формат (``.concat([...])``) — discovery-версия `_extract_from_concat`
    без key-фильтра: собирает ВСЕ key/value пары из каждого concat-массива сразу
    (один concat-вызов на mfe несёт сразу несколько key/value записей)."""
    result: dict[str, dict[str, Any]] = {}
    for m in _RE_CIAN_CONCAT.finditer(html):
        mfe = m.group("mfe")
        start = m.end()
        if start >= len(html) or html[start] != "[":
            start = html.find("[", m.end())
        if start < 0:
            continue
        arr = _balanced_array(html, start)
        if arr is None:
            continue
        try:
            items = json.loads(arr)
        except json.JSONDecodeError as exc:
            logger.debug("concat array parse failed for mfe=%s (discovery): %s", mfe, exc)
            continue
        for it in items:
            if isinstance(it, dict) and "key" in it:
                result.setdefault(mfe, {})[it["key"]] = it.get("value")
    return result


def extract_all_states(html: str) -> dict[str, dict[str, Any]]:
    """Returns {mfe_name: {key: parsed_value}} for all push()/concat() entries.

    Useful for discovery / debugging during scraper development. Bug fix (found live
    2026-07-04 while verifying #2435 bti_data persistence): this previously only
    scanned the legacy `.push({...})` pattern (`_RE_CIAN_PUSH`) — but Cian switched to
    `.concat([...])` in 2026-05 (#639), which `extract_state()` already handles
    (concat checked first, push as fallback) while this function never did. Result:
    on current-format pages this returned `{}` unconditionally, silently starving
    every `extract_all_states()` consumer (bti sister-state in detail.py, newbuilding
    sister-state in newbuilding.py) of real data. Mirrors extract_state()'s
    concat-first / push-fallback precedence.
    """
    result = _extract_all_from_concat(html)
    for match in _RE_CIAN_PUSH.finditer(html):
        mfe = match.group("mfe")
        key = match.group("key")
        if key in result.get(mfe, {}):
            continue  # concat already supplied this mfe/key — don't reparse/overwrite
        parsed = extract_state(html, mfe, key)
        if parsed is not None:
            result.setdefault(mfe, {})[key] = parsed
    return result