93 lines
3.2 KiB
Python
93 lines
3.2 KiB
Python
"""Shared utility для извлечения Cian Redux state из _cianConfig.
|
|
|
|
Pattern: window._cianConfig['<mfe>'].push({key:..., value:..., priority:..., filter:...})
|
|
Used by SERP / detail / newbuilding / valuation scrapers.
|
|
"""
|
|
import json
|
|
import logging
|
|
import re
|
|
from typing import Any
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Match both patterns Cian uses:
|
|
# 1. window._cianConfig['mfe'].push({key:..., value:..., priority:..., filter:...})
|
|
# 2. window._cianConfig['mfe'] = window._cianConfig['mfe'] || [];
|
|
# window._cianConfig['mfe'].push({...})
|
|
_RE_CIAN_PUSH = re.compile(
|
|
r"window\._cianConfig\['(?P<mfe>[\w-]+)'\]"
|
|
r"(?:\.push\(\s*|"
|
|
r"\s*=\s*window\._cianConfig\['[\w-]+'\]\s*\|\|\s*\[\];\s*"
|
|
r"window\._cianConfig\['[\w-]+'\]\.push\(\s*)"
|
|
r"\{\s*key:\s*['\"](?P<key>\w+)['\"]\s*,\s*"
|
|
r"value:\s*(?P<value>.+?)\s*,\s*"
|
|
r"priority:",
|
|
re.DOTALL,
|
|
)
|
|
|
|
|
|
def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, Any] | None:
|
|
"""Extract state.value where _cianConfig[<mfe>] entry.key matches.
|
|
|
|
Returns parsed JSON dict, or None if not found / parse failed.
|
|
Logs warning at debug level on parse failure for diagnostics.
|
|
"""
|
|
for match in _RE_CIAN_PUSH.finditer(html):
|
|
if match.group("mfe") != mfe or match.group("key") != key:
|
|
continue
|
|
|
|
value_str = match.group("value").strip()
|
|
|
|
# Strategy 1: direct JSON parse (value уже JSON-like)
|
|
try:
|
|
return json.loads(value_str)
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
# Strategy 2: JSON.parse("escaped string")
|
|
m_jp = re.match(r'JSON\.parse\(\s*"(.+?)"\s*\)\s*$', value_str, re.DOTALL)
|
|
if m_jp:
|
|
escaped = m_jp.group(1)
|
|
unescaped = (
|
|
escaped
|
|
.replace('\\"', '"')
|
|
.replace('\\\\', '\\')
|
|
.replace('\\n', '\n')
|
|
.replace('\\/', '/')
|
|
)
|
|
try:
|
|
return json.loads(unescaped)
|
|
except json.JSONDecodeError as exc:
|
|
logger.debug(
|
|
"JSON.parse() unescape failed for mfe=%s key=%s: %s", mfe, key, exc
|
|
)
|
|
continue
|
|
|
|
# Strategy 3: raw JS object literal — try demjson3 (optional dep)
|
|
try:
|
|
import demjson3 # type: ignore[import-untyped]
|
|
|
|
return demjson3.decode(value_str)
|
|
except ImportError:
|
|
logger.debug(
|
|
"demjson3 not installed; cannot parse raw JS object for mfe=%s key=%s", mfe, key
|
|
)
|
|
except Exception as exc:
|
|
logger.debug("demjson3 parse failed for mfe=%s key=%s: %s", mfe, key, exc)
|
|
|
|
return None
|
|
|
|
|
|
def extract_all_states(html: str) -> dict[str, dict[str, Any]]:
|
|
"""Returns {mfe_name: {key: parsed_value}} for all push() entries.
|
|
|
|
Useful for discovery / debugging during scraper development.
|
|
"""
|
|
result: dict[str, dict[str, Any]] = {}
|
|
for match in _RE_CIAN_PUSH.finditer(html):
|
|
mfe = match.group("mfe")
|
|
key = match.group("key")
|
|
parsed = extract_state(html, mfe, key)
|
|
if parsed is not None:
|
|
result.setdefault(mfe, {})[key] = parsed
|
|
return result
|