fix(tradein/cian): extract_all_states — discover new .concat() format, not just legacy .push() (#2438)

tradein-mvp/backend/tests/test_cian_state_parser.py

@@ -0,0 +1,137 @@
+"""Тесты для `scraper_kit.cian_state_parser` (extract_state / extract_all_states).
+
+Regression coverage for a bug found live 2026-07-04 while verifying #2435
+(bti_data persistence): `extract_all_states()` only ever scanned the legacy
+`.push({...})` `_cianConfig` format via `_RE_CIAN_PUSH`. Cian switched real pages
+to `.concat([...])` in 2026-05 (#639) — `extract_state()` was updated to handle
+both (concat-first, push-fallback), but `extract_all_states()` was never updated,
+so on every current-format page it silently returned `{}` — starving both of its
+consumers (`cian/detail.py`'s bti sister-state, `cian/newbuilding.py`'s sister-state)
+of real data, confirmed live against a real offer page (200 OK, 595KB HTML) where
+`extract_all_states()` returned zero top-level MFE keys despite `extract_state()`
+successfully parsing `defaultState` from the very same HTML.
+
+No test file for this module existed before (a stale `.pyc` from a deleted
+pre-scraper_kit-migration test was the only trace) — this is new coverage, not
+a rewrite.
+"""
+
+from __future__ import annotations
+
+from scraper_kit.cian_state_parser import extract_all_states, extract_state
+
+
+def _concat_push(mfe: str, entries: list[tuple[str, str]]) -> str:
+    """Строит HTML-фрагмент в НОВОМ (2026-05, #639) `.concat([...])` формате.
+
+    `entries` — список (key, raw_json_value) пар, ужe сериализованных в JSON-текст
+    (не эскейпленных дважды — `.concat([...])` несёт валидный JSON-массив как есть).
+    """
+    items = ",".join(f'{{"key":"{k}","value":{v},"priority":0}}' for k, v in entries)
+    return (
+        f"window._cianConfig['{mfe}'] = (window._cianConfig['{mfe}'] || [])" f".concat([{items}]);"
+    )
+
+
+def _push(mfe: str, key: str, value: str) -> str:
+    """Строит HTML-фрагмент в ЛЕГАСИ `.push({...})` формате."""
+    return f"window._cianConfig['{mfe}'].push({{key:'{key}', value:{value}, priority:0}});"
+
+
+def test_extract_all_states_finds_concat_format_mfe_and_keys():
+    """Regression: до фикса это возвращало {} на любой странице в новом формате."""
+    html = (
+        "<script>"
+        + _concat_push(
+            "frontend-offer-card",
+            [
+                ("defaultState", '{"offerData":{"offer":{"cianId":123}}}'),
+                ("bti", '{"houseData":{"seriesName":"1-464","entrances":4}}'),
+            ],
+        )
+        + "</script>"
+    )
+
+    result = extract_all_states(html)
+
+    assert "frontend-offer-card" in result
+    offer_card = result["frontend-offer-card"]
+    assert set(offer_card.keys()) == {"defaultState", "bti"}
+    assert offer_card["bti"]["houseData"]["seriesName"] == "1-464"
+    assert offer_card["bti"]["houseData"]["entrances"] == 4
+
+
+def test_extract_all_states_multiple_mfe_containers_concat_format():
+    """Несколько разных mfe-контейнеров в одном html — все должны попасть в результат
+    (регрессия на реальный кейс: живая страница вернула 6 top-level mfe-ключей)."""
+    html = (
+        "<script>"
+        + _concat_push("frontend-offer-card", [("defaultState", '{"a":1}')])
+        + _concat_push("header-frontend", [("version", '"1.2.3"')])
+        + _concat_push("frontend-footer", [("locale", '"ru"')])
+        + "</script>"
+    )
+
+    result = extract_all_states(html)
+
+    assert set(result.keys()) == {"frontend-offer-card", "header-frontend", "frontend-footer"}
+    assert result["header-frontend"]["version"] == "1.2.3"
+
+
+def test_extract_all_states_legacy_push_format_still_works():
+    """Fallback-путь (старый .push() формат) не должен был сломаться фиксом."""
+    html = (
+        "<script>"
+        + _push("frontend-offer-card", "defaultState", '{"offerData":{"offer":{"cianId":456}}}')
+        + _push("frontend-offer-card", "bti", '{"houseData":{"flatCount":58}}')
+        + "</script>"
+    )
+
+    result = extract_all_states(html)
+
+    assert "frontend-offer-card" in result
+    offer_card = result["frontend-offer-card"]
+    assert offer_card["defaultState"]["offerData"]["offer"]["cianId"] == 456
+    assert offer_card["bti"]["houseData"]["flatCount"] == 58
+
+
+def test_extract_all_states_mixed_concat_and_push_prefers_concat():
+    """Смешанный html (concat для одного mfe/key, push для другого/того же) —
+    concat не должен быть перезаписан push-фолбэком на тот же (mfe, key)."""
+    html = (
+        "<script>"
+        + _concat_push("frontend-offer-card", [("bti", '{"houseData":{"entrances":9}}')])
+        + _push("frontend-offer-card", "bti", '{"houseData":{"entrances":1}}')
+        + _push("header-frontend", "version", '"9.9.9"')
+        + "</script>"
+    )
+
+    result = extract_all_states(html)
+
+    # concat-значение (entrances=9) должно выиграть, push с тем же (mfe,key) — не перезаписывать
+    assert result["frontend-offer-card"]["bti"]["houseData"]["entrances"] == 9
+    # push-only mfe (header-frontend) всё равно должен попасть в результат
+    assert result["header-frontend"]["version"] == "9.9.9"
+
+
+def test_extract_all_states_no_cianconfig_returns_empty_dict():
+    """Страница без _cianConfig вообще (или сломанная) — пустой dict, не exception."""
+    assert extract_all_states("<html><body>no config here</body></html>") == {}
+
+
+def test_extract_state_concat_format_matches_extract_all_states_value():
+    """Sanity: extract_state() и extract_all_states() должны согласованно видеть
+    одно и то же значение для одного и того же (mfe, key) на concat-странице —
+    ключевой инвариант, который был нарушен багом (extract_state видел данные,
+    extract_all_states — нет)."""
+    html = (
+        "<script>"
+        + _concat_push("frontend-offer-card", [("bti", '{"houseData":{"seriesName":"X"}}')])
+        + "</script>"
+    )
+
+    single = extract_state(html, mfe="frontend-offer-card", key="bti")
+    all_states = extract_all_states(html)
+
+    assert single is not None
+    assert all_states["frontend-offer-card"]["bti"] == single

tradein-mvp/packages/scraper-kit/src/scraper_kit/cian_state_parser.py

@@ -145,15 +145,51 @@ def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, A
     return None
 
 
-def extract_all_states(html: str) -> dict[str, dict[str, Any]]:
-    """Returns {mfe_name: {key: parsed_value}} for all push() entries.
-
-    Useful for discovery / debugging during scraper development.
-    """
+def _extract_all_from_concat(html: str) -> dict[str, dict[str, Any]]:
+    """Новый формат (``.concat([...])``) — discovery-версия `_extract_from_concat`
+    без key-фильтра: собирает ВСЕ key/value пары из каждого concat-массива сразу
+    (один concat-вызов на mfe несёт сразу несколько key/value записей)."""
     result: dict[str, dict[str, Any]] = {}
+    for m in _RE_CIAN_CONCAT.finditer(html):
+        mfe = m.group("mfe")
+        start = m.end()
+        if start >= len(html) or html[start] != "[":
+            start = html.find("[", m.end())
+        if start < 0:
+            continue
+        arr = _balanced_array(html, start)
+        if arr is None:
+            continue
+        try:
+            items = json.loads(arr)
+        except json.JSONDecodeError as exc:
+            logger.debug("concat array parse failed for mfe=%s (discovery): %s", mfe, exc)
+            continue
+        for it in items:
+            if isinstance(it, dict) and "key" in it:
+                result.setdefault(mfe, {})[it["key"]] = it.get("value")
+    return result
+
+
+def extract_all_states(html: str) -> dict[str, dict[str, Any]]:
+    """Returns {mfe_name: {key: parsed_value}} for all push()/concat() entries.
+
+    Useful for discovery / debugging during scraper development. Bug fix (found live
+    2026-07-04 while verifying #2435 bti_data persistence): this previously only
+    scanned the legacy `.push({...})` pattern (`_RE_CIAN_PUSH`) — but Cian switched to
+    `.concat([...])` in 2026-05 (#639), which `extract_state()` already handles
+    (concat checked first, push as fallback) while this function never did. Result:
+    on current-format pages this returned `{}` unconditionally, silently starving
+    every `extract_all_states()` consumer (bti sister-state in detail.py, newbuilding
+    sister-state in newbuilding.py) of real data. Mirrors extract_state()'s
+    concat-first / push-fallback precedence.
+    """
+    result = _extract_all_from_concat(html)
     for match in _RE_CIAN_PUSH.finditer(html):
         mfe = match.group("mfe")
         key = match.group("key")
+        if key in result.get(mfe, {}):
+            continue  # concat already supplied this mfe/key — don't reparse/overwrite
         parsed = extract_state(html, mfe, key)
         if parsed is not None:
             result.setdefault(mfe, {})[key] = parsed