Merge pull request 'fix(tradein): cian fetch_newbuilding через BrowserFetcher (camoufox) — initialState под анти-ботом (#972)' (#1540) from fix/cian-newbuilding-browser-fetch into main

Reviewed-on: #1540

tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py

@@ -25,6 +25,7 @@ from typing import Any
 from curl_cffi.requests import AsyncSession  # type: ignore[import-untyped]
 
 from app.core.config import settings
+from app.services.scrapers.browser_fetcher import BrowserFetcher
 from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
 
 logger = logging.getLogger(__name__)
@@ -120,116 +121,98 @@ class NewbuildingEnrichment:
 async def fetch_newbuilding(
     zhk_url: str,
     *,
-    session: AsyncSession | None = None,
+    session: AsyncSession | None = None,  # kept for backward-compat; unused for page fetch
 ) -> NewbuildingEnrichment | None:
     """Fetch ЖК catalog page и extract все containers.
 
+    HTML страница ЖК теперь тянется через BrowserFetcher (camoufox), потому что
+    curl_cffi получает страницу без initialState под анти-ботом Cian (#972).
+    tradein-browser верифицирован живым: 1.17 MB, initialState присутствует.
+
     Args:
         zhk_url: e.g. 'https://zhk-ekaterininskiy-park-ekb-i.cian.ru/'
-        session: optional shared curl_cffi session (caller owns lifecycle)
+        session: зарезервирован для обратной совместимости с вызывающими;
+            больше не используется для fetch страницы ЖК. Resolve-функции
+            (resolve_cian_zhk_url_via_search) по-прежнему используют curl_cffi.
 
     Returns: NewbuildingEnrichment, or None если fetch / parse failed.
     """
-    close_session = False
-    if session is None:
-        # Mobile proxy wiring (#806 follow-up): Cian ЖК страница — datacenter-IP бан.
-        # proxy=None → прямое подключение (dev без прокси).
-        _proxy_url = settings.cian_proxy_url
-        _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
-        session = AsyncSession(
-            impersonate="chrome120",
-            timeout=30.0,
-            proxies=_proxies,
-            headers={
-                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
-            },
-        )
-        close_session = True
+    async with BrowserFetcher() as browser:
+        html = await browser.fetch(zhk_url)
 
-    try:
-        resp = await session.get(zhk_url, allow_redirects=True)
-        if resp.status_code != 200:
-            logger.warning("Cian newbuilding fetch %s → HTTP %d", zhk_url, resp.status_code)
-            return None
-        html = resp.text
+    mfe = "newbuilding-card-desktop-fichering-frontend"
 
-        mfe = "newbuilding-card-desktop-fichering-frontend"
+    nb_state = extract_state(html, mfe=mfe, key="initialState")
+    if nb_state is None:
+        logger.warning("Cian newbuilding %s: initialState extraction failed", zhk_url)
+        return None
 
-        nb_state = extract_state(html, mfe=mfe, key="initialState")
-        if nb_state is None:
-            logger.warning("Cian newbuilding %s: initialState extraction failed", zhk_url)
-            return None
+    # Primary newbuilding object: state.newbuilding (60 fields per schema sec 15.4)
+    nb = nb_state.get("newbuilding") or {}
+    if not isinstance(nb, dict):
+        nb = {}
 
-        # Primary newbuilding object: state.newbuilding (60 fields per schema sec 15.4)
-        nb = nb_state.get("newbuilding") or {}
-        if not isinstance(nb, dict):
-            nb = {}
+    result = NewbuildingEnrichment(
+        cian_internal_house_id=nb.get("id"),
+        cian_zhk_url=zhk_url,
+        name=nb.get("name") or nb.get("displayName"),
+        address=_extract_address(nb),
+        year_built=nb.get("yearBuilt") or nb.get("buildYear"),
+        deadline_year=_extract_deadline_year(nb),
+        floors_count_min=nb.get("floorsCountMin"),
+        floors_count_max=nb.get("floorsCountMax"),
+        building_class=nb.get("newbuildingClassName") or nb.get("buildingClass"),
+        management_company=nb.get("managementCompany"),
+        transport_accessibility_rate=_extract_transport_rate(nb),
+        advantages=_extract_advantages(nb),
+        builders=_safe_list(nb.get("builders")),
+        banks=_safe_list(nb.get("banks")),
+        houses_by_turn=_safe_list(nb.get("housesByTurn")),
+        raw_state=nb_state,
+    )
 
-        result = NewbuildingEnrichment(
-            cian_internal_house_id=nb.get("id"),
-            cian_zhk_url=zhk_url,
-            name=nb.get("name") or nb.get("displayName"),
-            address=_extract_address(nb),
-            year_built=nb.get("yearBuilt") or nb.get("buildYear"),
-            deadline_year=_extract_deadline_year(nb),
-            floors_count_min=nb.get("floorsCountMin"),
-            floors_count_max=nb.get("floorsCountMax"),
-            building_class=nb.get("newbuildingClassName") or nb.get("buildingClass"),
-            management_company=nb.get("managementCompany"),
-            transport_accessibility_rate=_extract_transport_rate(nb),
-            advantages=_extract_advantages(nb),
-            builders=_safe_list(nb.get("builders")),
-            banks=_safe_list(nb.get("banks")),
-            houses_by_turn=_safe_list(nb.get("housesByTurn")),
-            raw_state=nb_state,
-        )
+    # Sister containers — all live under the same MFE initialState top-level keys
+    # (not separate push() entries) per schema sec 15.3
+    all_states = extract_all_states(html)
+    mfe_states = all_states.get(mfe, {})
+    # initialState top-level keys contain sister data: realtyValuation, reliability, offers
+    top = nb_state
 
-        # Sister containers — all live under the same MFE initialState top-level keys
-        # (not separate push() entries) per schema sec 15.3
-        all_states = extract_all_states(html)
-        mfe_states = all_states.get(mfe, {})
-        # initialState top-level keys contain sister data: realtyValuation, reliability, offers
-        top = nb_state
+    # realtyValuation 7-month chart (Stage 6 main deliverable)
+    # Located at initialState.realtyValuation (sec 15.5)
+    rv_state = top.get("realtyValuation") or mfe_states.get("realtyValuation")
+    if rv_state and isinstance(rv_state, dict):
+        result.realty_valuation_chart = _extract_chart(rv_state)
+        result.raw_sister_states["realtyValuation"] = rv_state
 
-        # realtyValuation 7-month chart (Stage 6 main deliverable)
-        # Located at initialState.realtyValuation (sec 15.5)
-        rv_state = top.get("realtyValuation") or mfe_states.get("realtyValuation")
-        if rv_state and isinstance(rv_state, dict):
-            result.realty_valuation_chart = _extract_chart(rv_state)
-            result.raw_sister_states["realtyValuation"] = rv_state
+    # reliability (наш.дом.рф) — located at initialState.reliability (sec 15.6)
+    rel_state = top.get("reliability") or mfe_states.get("reliability")
+    if rel_state and isinstance(rel_state, dict):
+        result.reliability_checks = _extract_reliability_checks(rel_state)
+        result.raw_sister_states["reliability"] = rel_state
 
-        # reliability (наш.дом.рф) — located at initialState.reliability (sec 15.6)
-        rel_state = top.get("reliability") or mfe_states.get("reliability")
-        if rel_state and isinstance(rel_state, dict):
-            result.reliability_checks = _extract_reliability_checks(rel_state)
-            result.raw_sister_states["reliability"] = rel_state
+    # reviews — located at initialState or separate state
+    reviews_state = top.get("reviews") or mfe_states.get("reviews")
+    if reviews_state and isinstance(reviews_state, dict):
+        items = reviews_state.get("items") or reviews_state.get("data") or []
+        result.reviews = items if isinstance(items, list) else []
+        result.raw_sister_states["reviews"] = reviews_state
 
-        # reviews — located at initialState or separate state
-        reviews_state = top.get("reviews") or mfe_states.get("reviews")
-        if reviews_state and isinstance(reviews_state, dict):
-            items = reviews_state.get("items") or reviews_state.get("data") or []
-            result.reviews = items if isinstance(items, list) else []
-            result.raw_sister_states["reviews"] = reviews_state
+    # offers (grouped by roomType) — located at initialState.offers (sec 15.7)
+    offers_state = top.get("offers") or mfe_states.get("offers") or mfe_states.get("miniSerp")
+    if offers_state and isinstance(offers_state, dict):
+        result.nested_offers = _extract_nested_offers(offers_state)
+        result.raw_sister_states["offers"] = offers_state
 
-        # offers (grouped by roomType) — located at initialState.offers (sec 15.7)
-        offers_state = top.get("offers") or mfe_states.get("offers") or mfe_states.get("miniSerp")
-        if offers_state and isinstance(offers_state, dict):
-            result.nested_offers = _extract_nested_offers(offers_state)
-            result.raw_sister_states["offers"] = offers_state
-
-        logger.info(
-            "Cian newbuilding %s parsed: id=%s chart=%d reliability=%d offers=%d",
-            zhk_url,
-            result.cian_internal_house_id,
-            len(result.realty_valuation_chart),
-            len(result.reliability_checks),
-            len(result.nested_offers),
-        )
-        return result
-    finally:
-        if close_session:
-            await session.close()
+    logger.info(
+        "Cian newbuilding %s parsed: id=%s chart=%d reliability=%d offers=%d",
+        zhk_url,
+        result.cian_internal_house_id,
+        len(result.realty_valuation_chart),
+        len(result.reliability_checks),
+        len(result.nested_offers),
+    )
+    return result
 
 
 # ---- private extractors ----

tradein-mvp/backend/tests/test_cian_newbuilding.py

@@ -1,4 +1,5 @@
 """Tests for cian_newbuilding scraper (Stage 6)."""
+
 from __future__ import annotations
 
 import pytest
@@ -247,42 +248,72 @@ def test_dataclass_defaults():
 
 
 # ---- async integration-style tests ----
+# Transport: fetch_newbuilding теперь использует BrowserFetcher (camoufox) вместо
+# curl_cffi AsyncSession (#972). Мокируем BrowserFetcher.fetch через monkeypatch.
+
+
+def _make_browser_fetcher_mock(html: str, monkeypatch: pytest.MonkeyPatch) -> None:
+    """Мокирует BrowserFetcher так, чтобы fetch() возвращал заданный html."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    mock_fetcher = MagicMock()
+    mock_fetcher.fetch = AsyncMock(return_value=html)
+    mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
+    mock_fetcher.__aexit__ = AsyncMock(return_value=None)
+
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_newbuilding.BrowserFetcher",
+        lambda: mock_fetcher,
+    )
 
 
 @pytest.mark.asyncio
 async def test_fetch_newbuilding_returns_none_on_no_state(monkeypatch):
     """Если state extraction fails → return None gracefully."""
+    _make_browser_fetcher_mock("<html></html>", monkeypatch)
     monkeypatch.setattr(
         "app.services.scrapers.cian_newbuilding.extract_state",
         lambda html, mfe, key: None,
     )
-    from unittest.mock import AsyncMock, MagicMock
 
-    session = MagicMock()
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html></html>"
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
+    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
     assert result is None
 
 
 @pytest.mark.asyncio
-async def test_fetch_newbuilding_returns_none_on_http_error(monkeypatch):
-    """HTTP non-200 → return None gracefully."""
+async def test_fetch_newbuilding_browser_error_propagates(monkeypatch):
+    """BrowserFetcher.fetch raises → fetch_newbuilding propagates the exception."""
     from unittest.mock import AsyncMock, MagicMock
 
-    session = MagicMock()
-    response = MagicMock()
-    response.status_code = 403
-    response.text = "Forbidden"
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
+    mock_fetcher = MagicMock()
+    mock_fetcher.fetch = AsyncMock(side_effect=RuntimeError("browser timeout"))
+    mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
+    mock_fetcher.__aexit__ = AsyncMock(return_value=None)
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_newbuilding.BrowserFetcher",
+        lambda: mock_fetcher,
+    )
 
-    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
+    with pytest.raises(RuntimeError, match="browser timeout"):
+        await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
+
+
+@pytest.mark.asyncio
+async def test_fetch_newbuilding_session_param_ignored(monkeypatch):
+    """session= param принимается для backward-compat, но не используется для page-fetch."""
+    _make_browser_fetcher_mock("<html></html>", monkeypatch)
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_newbuilding.extract_state",
+        lambda html, mfe, key: None,
+    )
+
+    from unittest.mock import MagicMock
+
+    sentinel_session = MagicMock(name="should_not_be_called")
+    # Передаём session= — должно работать без ошибок, session.get НЕ вызывается
+    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=sentinel_session)
     assert result is None
+    sentinel_session.get.assert_not_called()
 
 
 @pytest.mark.asyncio
@@ -318,6 +349,7 @@ async def test_fetch_newbuilding_parses_state(monkeypatch):
         },
     }
 
+    _make_browser_fetcher_mock("<html>fake</html>", monkeypatch)
     monkeypatch.setattr(
         "app.services.scrapers.cian_newbuilding.extract_state",
         lambda html, mfe, key: fake_state,
@@ -327,16 +359,7 @@ async def test_fetch_newbuilding_parses_state(monkeypatch):
         lambda html: {},
     )
 
-    from unittest.mock import AsyncMock, MagicMock
-
-    session = MagicMock()
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html>fake</html>"
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
+    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
 
     assert result is not None
     assert result.cian_internal_house_id == 54016