From 7833c354e959529fe27d1c6d1f9464dc246622db Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 23:45:55 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein/cian):=20extract=5Fall=5Fstates=20?= =?UTF-8?q?=E2=80=94=20discover=20new=20.concat()=20format,=20not=20just?= =?UTF-8?q?=20legacy=20.push()?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found live 2026-07-04 while verifying #2435 (bti_data persistence): a real Cian offer page (200 OK, 595KB) parsed fine via extract_state() (which already tries .concat() first, .push() as fallback per the #639 2026-05 format drift), but extract_all_states() returned {} — it only ever scanned the legacy .push() pattern via _RE_CIAN_PUSH, never updated when .concat() became the live format. This silently starved both of its consumers of real data on every current-format page: cian/detail.py's bti sister-state (the exact gap #2435 built persistence for, which could never receive data) and cian/newbuilding.py's sister-state extraction. Confirmed via live re-fetch after the fix: extract_all_states() now returns 6 real MFE containers instead of an empty dict. No test file previously existed for cian_state_parser.py (a stale .pyc from a pre-scraper_kit-migration test was the only trace) — added tests/test_cian_state_parser.py covering concat discovery, legacy push fallback, mixed-format precedence, and the extract_state()/extract_all_states() consistency invariant that the bug violated. Full backend suite: 2345 passed, 6 skipped, 1 pre-existing unrelated failure (test_search_cache_hit, verified via git stash against clean main). Refs #2435 --- .../backend/tests/test_cian_state_parser.py | 137 ++++++++++++++++++ .../src/scraper_kit/cian_state_parser.py | 46 +++++- 2 files changed, 178 insertions(+), 5 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_cian_state_parser.py diff --git a/tradein-mvp/backend/tests/test_cian_state_parser.py b/tradein-mvp/backend/tests/test_cian_state_parser.py new file mode 100644 index 00000000..e8d7e4c4 --- /dev/null +++ b/tradein-mvp/backend/tests/test_cian_state_parser.py @@ -0,0 +1,137 @@ +"""Тесты для `scraper_kit.cian_state_parser` (extract_state / extract_all_states). + +Regression coverage for a bug found live 2026-07-04 while verifying #2435 +(bti_data persistence): `extract_all_states()` only ever scanned the legacy +`.push({...})` `_cianConfig` format via `_RE_CIAN_PUSH`. Cian switched real pages +to `.concat([...])` in 2026-05 (#639) — `extract_state()` was updated to handle +both (concat-first, push-fallback), but `extract_all_states()` was never updated, +so on every current-format page it silently returned `{}` — starving both of its +consumers (`cian/detail.py`'s bti sister-state, `cian/newbuilding.py`'s sister-state) +of real data, confirmed live against a real offer page (200 OK, 595KB HTML) where +`extract_all_states()` returned zero top-level MFE keys despite `extract_state()` +successfully parsing `defaultState` from the very same HTML. + +No test file for this module existed before (a stale `.pyc` from a deleted +pre-scraper_kit-migration test was the only trace) — this is new coverage, not +a rewrite. +""" + +from __future__ import annotations + +from scraper_kit.cian_state_parser import extract_all_states, extract_state + + +def _concat_push(mfe: str, entries: list[tuple[str, str]]) -> str: + """Строит HTML-фрагмент в НОВОМ (2026-05, #639) `.concat([...])` формате. + + `entries` — список (key, raw_json_value) пар, ужe сериализованных в JSON-текст + (не эскейпленных дважды — `.concat([...])` несёт валидный JSON-массив как есть). + """ + items = ",".join(f'{{"key":"{k}","value":{v},"priority":0}}' for k, v in entries) + return ( + f"window._cianConfig['{mfe}'] = (window._cianConfig['{mfe}'] || [])" f".concat([{items}]);" + ) + + +def _push(mfe: str, key: str, value: str) -> str: + """Строит HTML-фрагмент в ЛЕГАСИ `.push({...})` формате.""" + return f"window._cianConfig['{mfe}'].push({{key:'{key}', value:{value}, priority:0}});" + + +def test_extract_all_states_finds_concat_format_mfe_and_keys(): + """Regression: до фикса это возвращало {} на любой странице в новом формате.""" + html = ( + "" + ) + + result = extract_all_states(html) + + assert "frontend-offer-card" in result + offer_card = result["frontend-offer-card"] + assert set(offer_card.keys()) == {"defaultState", "bti"} + assert offer_card["bti"]["houseData"]["seriesName"] == "1-464" + assert offer_card["bti"]["houseData"]["entrances"] == 4 + + +def test_extract_all_states_multiple_mfe_containers_concat_format(): + """Несколько разных mfe-контейнеров в одном html — все должны попасть в результат + (регрессия на реальный кейс: живая страница вернула 6 top-level mfe-ключей).""" + html = ( + "" + ) + + result = extract_all_states(html) + + assert set(result.keys()) == {"frontend-offer-card", "header-frontend", "frontend-footer"} + assert result["header-frontend"]["version"] == "1.2.3" + + +def test_extract_all_states_legacy_push_format_still_works(): + """Fallback-путь (старый .push() формат) не должен был сломаться фиксом.""" + html = ( + "" + ) + + result = extract_all_states(html) + + assert "frontend-offer-card" in result + offer_card = result["frontend-offer-card"] + assert offer_card["defaultState"]["offerData"]["offer"]["cianId"] == 456 + assert offer_card["bti"]["houseData"]["flatCount"] == 58 + + +def test_extract_all_states_mixed_concat_and_push_prefers_concat(): + """Смешанный html (concat для одного mfe/key, push для другого/того же) — + concat не должен быть перезаписан push-фолбэком на тот же (mfe, key).""" + html = ( + "" + ) + + result = extract_all_states(html) + + # concat-значение (entrances=9) должно выиграть, push с тем же (mfe,key) — не перезаписывать + assert result["frontend-offer-card"]["bti"]["houseData"]["entrances"] == 9 + # push-only mfe (header-frontend) всё равно должен попасть в результат + assert result["header-frontend"]["version"] == "9.9.9" + + +def test_extract_all_states_no_cianconfig_returns_empty_dict(): + """Страница без _cianConfig вообще (или сломанная) — пустой dict, не exception.""" + assert extract_all_states("no config here") == {} + + +def test_extract_state_concat_format_matches_extract_all_states_value(): + """Sanity: extract_state() и extract_all_states() должны согласованно видеть + одно и то же значение для одного и того же (mfe, key) на concat-странице — + ключевой инвариант, который был нарушен багом (extract_state видел данные, + extract_all_states — нет).""" + html = ( + "" + ) + + single = extract_state(html, mfe="frontend-offer-card", key="bti") + all_states = extract_all_states(html) + + assert single is not None + assert all_states["frontend-offer-card"]["bti"] == single diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/cian_state_parser.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/cian_state_parser.py index f6a46a7a..156a855d 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/cian_state_parser.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/cian_state_parser.py @@ -145,15 +145,51 @@ def extract_state(html: str, mfe: str, key: str = "initialState") -> dict[str, A return None -def extract_all_states(html: str) -> dict[str, dict[str, Any]]: - """Returns {mfe_name: {key: parsed_value}} for all push() entries. - - Useful for discovery / debugging during scraper development. - """ +def _extract_all_from_concat(html: str) -> dict[str, dict[str, Any]]: + """Новый формат (``.concat([...])``) — discovery-версия `_extract_from_concat` + без key-фильтра: собирает ВСЕ key/value пары из каждого concat-массива сразу + (один concat-вызов на mfe несёт сразу несколько key/value записей).""" result: dict[str, dict[str, Any]] = {} + for m in _RE_CIAN_CONCAT.finditer(html): + mfe = m.group("mfe") + start = m.end() + if start >= len(html) or html[start] != "[": + start = html.find("[", m.end()) + if start < 0: + continue + arr = _balanced_array(html, start) + if arr is None: + continue + try: + items = json.loads(arr) + except json.JSONDecodeError as exc: + logger.debug("concat array parse failed for mfe=%s (discovery): %s", mfe, exc) + continue + for it in items: + if isinstance(it, dict) and "key" in it: + result.setdefault(mfe, {})[it["key"]] = it.get("value") + return result + + +def extract_all_states(html: str) -> dict[str, dict[str, Any]]: + """Returns {mfe_name: {key: parsed_value}} for all push()/concat() entries. + + Useful for discovery / debugging during scraper development. Bug fix (found live + 2026-07-04 while verifying #2435 bti_data persistence): this previously only + scanned the legacy `.push({...})` pattern (`_RE_CIAN_PUSH`) — but Cian switched to + `.concat([...])` in 2026-05 (#639), which `extract_state()` already handles + (concat checked first, push as fallback) while this function never did. Result: + on current-format pages this returned `{}` unconditionally, silently starving + every `extract_all_states()` consumer (bti sister-state in detail.py, newbuilding + sister-state in newbuilding.py) of real data. Mirrors extract_state()'s + concat-first / push-fallback precedence. + """ + result = _extract_all_from_concat(html) for match in _RE_CIAN_PUSH.finditer(html): mfe = match.group("mfe") key = match.group("key") + if key in result.get(mfe, {}): + continue # concat already supplied this mfe/key — don't reparse/overwrite parsed = extract_state(html, mfe, key) if parsed is not None: result.setdefault(mfe, {})[key] = parsed