From 480fbba3ca489f4f82654aa69ac27c804fee052a Mon Sep 17 00:00:00 2001 From: bot-backend Date: Mon, 15 Jun 2026 18:19:32 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein):=20cian=20fetch=5Fnewbuilding=20?= =?UTF-8?q?=D1=87=D0=B5=D1=80=D0=B5=D0=B7=20BrowserFetcher=20(camoufox)=20?= =?UTF-8?q?=E2=80=94=20initialState=20=D0=BF=D0=BE=D0=B4=20=D0=B0=D0=BD?= =?UTF-8?q?=D1=82=D0=B8-=D0=B1=D0=BE=D1=82=D0=BE=D0=BC=20(#972)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit curl_cffi получает zhk-*.cian.ru без initialState; tradein-browser (camoufox) тянет страницу с 1.17MB initialState. Заменяем транспорт только для page-fetch; session= param сохранён для backward-compat вызывающих. resolve_cian_zhk_url_via_search остаётся на curl_cffi SERP. Тесты: 21 passed. --- .../app/services/scrapers/cian_newbuilding.py | 167 ++++++++---------- .../backend/tests/test_cian_newbuilding.py | 79 ++++++--- 2 files changed, 126 insertions(+), 120 deletions(-) diff --git a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py index bca86175..9ce09f20 100644 --- a/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py +++ b/tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py @@ -25,6 +25,7 @@ from typing import Any from curl_cffi.requests import AsyncSession # type: ignore[import-untyped] from app.core.config import settings +from app.services.scrapers.browser_fetcher import BrowserFetcher from app.services.scrapers.cian_state_parser import extract_all_states, extract_state logger = logging.getLogger(__name__) @@ -120,116 +121,98 @@ class NewbuildingEnrichment: async def fetch_newbuilding( zhk_url: str, *, - session: AsyncSession | None = None, + session: AsyncSession | None = None, # kept for backward-compat; unused for page fetch ) -> NewbuildingEnrichment | None: """Fetch ЖК catalog page и extract все containers. + HTML страница ЖК теперь тянется через BrowserFetcher (camoufox), потому что + curl_cffi получает страницу без initialState под анти-ботом Cian (#972). + tradein-browser верифицирован живым: 1.17 MB, initialState присутствует. + Args: zhk_url: e.g. 'https://zhk-ekaterininskiy-park-ekb-i.cian.ru/' - session: optional shared curl_cffi session (caller owns lifecycle) + session: зарезервирован для обратной совместимости с вызывающими; + больше не используется для fetch страницы ЖК. Resolve-функции + (resolve_cian_zhk_url_via_search) по-прежнему используют curl_cffi. Returns: NewbuildingEnrichment, or None если fetch / parse failed. """ - close_session = False - if session is None: - # Mobile proxy wiring (#806 follow-up): Cian ЖК страница — datacenter-IP бан. - # proxy=None → прямое подключение (dev без прокси). - _proxy_url = settings.cian_proxy_url - _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None - session = AsyncSession( - impersonate="chrome120", - timeout=30.0, - proxies=_proxies, - headers={ - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", - "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8", - }, - ) - close_session = True + async with BrowserFetcher() as browser: + html = await browser.fetch(zhk_url) - try: - resp = await session.get(zhk_url, allow_redirects=True) - if resp.status_code != 200: - logger.warning("Cian newbuilding fetch %s → HTTP %d", zhk_url, resp.status_code) - return None - html = resp.text + mfe = "newbuilding-card-desktop-fichering-frontend" - mfe = "newbuilding-card-desktop-fichering-frontend" + nb_state = extract_state(html, mfe=mfe, key="initialState") + if nb_state is None: + logger.warning("Cian newbuilding %s: initialState extraction failed", zhk_url) + return None - nb_state = extract_state(html, mfe=mfe, key="initialState") - if nb_state is None: - logger.warning("Cian newbuilding %s: initialState extraction failed", zhk_url) - return None + # Primary newbuilding object: state.newbuilding (60 fields per schema sec 15.4) + nb = nb_state.get("newbuilding") or {} + if not isinstance(nb, dict): + nb = {} - # Primary newbuilding object: state.newbuilding (60 fields per schema sec 15.4) - nb = nb_state.get("newbuilding") or {} - if not isinstance(nb, dict): - nb = {} + result = NewbuildingEnrichment( + cian_internal_house_id=nb.get("id"), + cian_zhk_url=zhk_url, + name=nb.get("name") or nb.get("displayName"), + address=_extract_address(nb), + year_built=nb.get("yearBuilt") or nb.get("buildYear"), + deadline_year=_extract_deadline_year(nb), + floors_count_min=nb.get("floorsCountMin"), + floors_count_max=nb.get("floorsCountMax"), + building_class=nb.get("newbuildingClassName") or nb.get("buildingClass"), + management_company=nb.get("managementCompany"), + transport_accessibility_rate=_extract_transport_rate(nb), + advantages=_extract_advantages(nb), + builders=_safe_list(nb.get("builders")), + banks=_safe_list(nb.get("banks")), + houses_by_turn=_safe_list(nb.get("housesByTurn")), + raw_state=nb_state, + ) - result = NewbuildingEnrichment( - cian_internal_house_id=nb.get("id"), - cian_zhk_url=zhk_url, - name=nb.get("name") or nb.get("displayName"), - address=_extract_address(nb), - year_built=nb.get("yearBuilt") or nb.get("buildYear"), - deadline_year=_extract_deadline_year(nb), - floors_count_min=nb.get("floorsCountMin"), - floors_count_max=nb.get("floorsCountMax"), - building_class=nb.get("newbuildingClassName") or nb.get("buildingClass"), - management_company=nb.get("managementCompany"), - transport_accessibility_rate=_extract_transport_rate(nb), - advantages=_extract_advantages(nb), - builders=_safe_list(nb.get("builders")), - banks=_safe_list(nb.get("banks")), - houses_by_turn=_safe_list(nb.get("housesByTurn")), - raw_state=nb_state, - ) + # Sister containers — all live under the same MFE initialState top-level keys + # (not separate push() entries) per schema sec 15.3 + all_states = extract_all_states(html) + mfe_states = all_states.get(mfe, {}) + # initialState top-level keys contain sister data: realtyValuation, reliability, offers + top = nb_state - # Sister containers — all live under the same MFE initialState top-level keys - # (not separate push() entries) per schema sec 15.3 - all_states = extract_all_states(html) - mfe_states = all_states.get(mfe, {}) - # initialState top-level keys contain sister data: realtyValuation, reliability, offers - top = nb_state + # realtyValuation 7-month chart (Stage 6 main deliverable) + # Located at initialState.realtyValuation (sec 15.5) + rv_state = top.get("realtyValuation") or mfe_states.get("realtyValuation") + if rv_state and isinstance(rv_state, dict): + result.realty_valuation_chart = _extract_chart(rv_state) + result.raw_sister_states["realtyValuation"] = rv_state - # realtyValuation 7-month chart (Stage 6 main deliverable) - # Located at initialState.realtyValuation (sec 15.5) - rv_state = top.get("realtyValuation") or mfe_states.get("realtyValuation") - if rv_state and isinstance(rv_state, dict): - result.realty_valuation_chart = _extract_chart(rv_state) - result.raw_sister_states["realtyValuation"] = rv_state + # reliability (наш.дом.рф) — located at initialState.reliability (sec 15.6) + rel_state = top.get("reliability") or mfe_states.get("reliability") + if rel_state and isinstance(rel_state, dict): + result.reliability_checks = _extract_reliability_checks(rel_state) + result.raw_sister_states["reliability"] = rel_state - # reliability (наш.дом.рф) — located at initialState.reliability (sec 15.6) - rel_state = top.get("reliability") or mfe_states.get("reliability") - if rel_state and isinstance(rel_state, dict): - result.reliability_checks = _extract_reliability_checks(rel_state) - result.raw_sister_states["reliability"] = rel_state + # reviews — located at initialState or separate state + reviews_state = top.get("reviews") or mfe_states.get("reviews") + if reviews_state and isinstance(reviews_state, dict): + items = reviews_state.get("items") or reviews_state.get("data") or [] + result.reviews = items if isinstance(items, list) else [] + result.raw_sister_states["reviews"] = reviews_state - # reviews — located at initialState or separate state - reviews_state = top.get("reviews") or mfe_states.get("reviews") - if reviews_state and isinstance(reviews_state, dict): - items = reviews_state.get("items") or reviews_state.get("data") or [] - result.reviews = items if isinstance(items, list) else [] - result.raw_sister_states["reviews"] = reviews_state + # offers (grouped by roomType) — located at initialState.offers (sec 15.7) + offers_state = top.get("offers") or mfe_states.get("offers") or mfe_states.get("miniSerp") + if offers_state and isinstance(offers_state, dict): + result.nested_offers = _extract_nested_offers(offers_state) + result.raw_sister_states["offers"] = offers_state - # offers (grouped by roomType) — located at initialState.offers (sec 15.7) - offers_state = top.get("offers") or mfe_states.get("offers") or mfe_states.get("miniSerp") - if offers_state and isinstance(offers_state, dict): - result.nested_offers = _extract_nested_offers(offers_state) - result.raw_sister_states["offers"] = offers_state - - logger.info( - "Cian newbuilding %s parsed: id=%s chart=%d reliability=%d offers=%d", - zhk_url, - result.cian_internal_house_id, - len(result.realty_valuation_chart), - len(result.reliability_checks), - len(result.nested_offers), - ) - return result - finally: - if close_session: - await session.close() + logger.info( + "Cian newbuilding %s parsed: id=%s chart=%d reliability=%d offers=%d", + zhk_url, + result.cian_internal_house_id, + len(result.realty_valuation_chart), + len(result.reliability_checks), + len(result.nested_offers), + ) + return result # ---- private extractors ---- diff --git a/tradein-mvp/backend/tests/test_cian_newbuilding.py b/tradein-mvp/backend/tests/test_cian_newbuilding.py index e5d02b65..e57fc5b6 100644 --- a/tradein-mvp/backend/tests/test_cian_newbuilding.py +++ b/tradein-mvp/backend/tests/test_cian_newbuilding.py @@ -1,4 +1,5 @@ """Tests for cian_newbuilding scraper (Stage 6).""" + from __future__ import annotations import pytest @@ -247,42 +248,72 @@ def test_dataclass_defaults(): # ---- async integration-style tests ---- +# Transport: fetch_newbuilding теперь использует BrowserFetcher (camoufox) вместо +# curl_cffi AsyncSession (#972). Мокируем BrowserFetcher.fetch через monkeypatch. + + +def _make_browser_fetcher_mock(html: str, monkeypatch: pytest.MonkeyPatch) -> None: + """Мокирует BrowserFetcher так, чтобы fetch() возвращал заданный html.""" + from unittest.mock import AsyncMock, MagicMock + + mock_fetcher = MagicMock() + mock_fetcher.fetch = AsyncMock(return_value=html) + mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher) + mock_fetcher.__aexit__ = AsyncMock(return_value=None) + + monkeypatch.setattr( + "app.services.scrapers.cian_newbuilding.BrowserFetcher", + lambda: mock_fetcher, + ) @pytest.mark.asyncio async def test_fetch_newbuilding_returns_none_on_no_state(monkeypatch): """Если state extraction fails → return None gracefully.""" + _make_browser_fetcher_mock("", monkeypatch) monkeypatch.setattr( "app.services.scrapers.cian_newbuilding.extract_state", lambda html, mfe, key: None, ) - from unittest.mock import AsyncMock, MagicMock - session = MagicMock() - response = MagicMock() - response.status_code = 200 - response.text = "" - session.get = AsyncMock(return_value=response) - session.close = AsyncMock() - - result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session) + result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/") assert result is None @pytest.mark.asyncio -async def test_fetch_newbuilding_returns_none_on_http_error(monkeypatch): - """HTTP non-200 → return None gracefully.""" +async def test_fetch_newbuilding_browser_error_propagates(monkeypatch): + """BrowserFetcher.fetch raises → fetch_newbuilding propagates the exception.""" from unittest.mock import AsyncMock, MagicMock - session = MagicMock() - response = MagicMock() - response.status_code = 403 - response.text = "Forbidden" - session.get = AsyncMock(return_value=response) - session.close = AsyncMock() + mock_fetcher = MagicMock() + mock_fetcher.fetch = AsyncMock(side_effect=RuntimeError("browser timeout")) + mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher) + mock_fetcher.__aexit__ = AsyncMock(return_value=None) + monkeypatch.setattr( + "app.services.scrapers.cian_newbuilding.BrowserFetcher", + lambda: mock_fetcher, + ) - result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session) + with pytest.raises(RuntimeError, match="browser timeout"): + await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/") + + +@pytest.mark.asyncio +async def test_fetch_newbuilding_session_param_ignored(monkeypatch): + """session= param принимается для backward-compat, но не используется для page-fetch.""" + _make_browser_fetcher_mock("", monkeypatch) + monkeypatch.setattr( + "app.services.scrapers.cian_newbuilding.extract_state", + lambda html, mfe, key: None, + ) + + from unittest.mock import MagicMock + + sentinel_session = MagicMock(name="should_not_be_called") + # Передаём session= — должно работать без ошибок, session.get НЕ вызывается + result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=sentinel_session) assert result is None + sentinel_session.get.assert_not_called() @pytest.mark.asyncio @@ -318,6 +349,7 @@ async def test_fetch_newbuilding_parses_state(monkeypatch): }, } + _make_browser_fetcher_mock("fake", monkeypatch) monkeypatch.setattr( "app.services.scrapers.cian_newbuilding.extract_state", lambda html, mfe, key: fake_state, @@ -327,16 +359,7 @@ async def test_fetch_newbuilding_parses_state(monkeypatch): lambda html: {}, ) - from unittest.mock import AsyncMock, MagicMock - - session = MagicMock() - response = MagicMock() - response.status_code = 200 - response.text = "fake" - session.get = AsyncMock(return_value=response) - session.close = AsyncMock() - - result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session) + result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/") assert result is not None assert result.cian_internal_house_id == 54016