Merge pull request 'fix(tradein): cian fetch_newbuilding через BrowserFetcher (camoufox) — initialState под анти-ботом (#972)' (#1540) from fix/cian-newbuilding-browser-fetch into main

Reviewed-on: #1540

tradein-mvp/backend/app/services/scrapers/cian_newbuilding.py

@@ -25,6 +25,7 @@ from typing import Any
 from curl_cffi.requests import AsyncSession  # type: ignore[import-untyped]
 
 from app.core.config import settings
+from app.services.scrapers.browser_fetcher import BrowserFetcher
 from app.services.scrapers.cian_state_parser import extract_all_states, extract_state
 
 logger = logging.getLogger(__name__)
@@ -120,39 +121,24 @@ class NewbuildingEnrichment:
 async def fetch_newbuilding(
     zhk_url: str,
     *,
-    session: AsyncSession | None = None,
+    session: AsyncSession | None = None,  # kept for backward-compat; unused for page fetch
 ) -> NewbuildingEnrichment | None:
     """Fetch ЖК catalog page и extract все containers.
 
+    HTML страница ЖК теперь тянется через BrowserFetcher (camoufox), потому что
+    curl_cffi получает страницу без initialState под анти-ботом Cian (#972).
+    tradein-browser верифицирован живым: 1.17 MB, initialState присутствует.
+
     Args:
         zhk_url: e.g. 'https://zhk-ekaterininskiy-park-ekb-i.cian.ru/'
-        session: optional shared curl_cffi session (caller owns lifecycle)
+        session: зарезервирован для обратной совместимости с вызывающими;
+            больше не используется для fetch страницы ЖК. Resolve-функции
+            (resolve_cian_zhk_url_via_search) по-прежнему используют curl_cffi.
 
     Returns: NewbuildingEnrichment, or None если fetch / parse failed.
     """
-    close_session = False
+    async with BrowserFetcher() as browser:
-    if session is None:
+        html = await browser.fetch(zhk_url)
-        # Mobile proxy wiring (#806 follow-up): Cian ЖК страница — datacenter-IP бан.
-        # proxy=None → прямое подключение (dev без прокси).
-        _proxy_url = settings.cian_proxy_url
-        _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
-        session = AsyncSession(
-            impersonate="chrome120",
-            timeout=30.0,
-            proxies=_proxies,
-            headers={
-                "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
-            },
-        )
-        close_session = True
-
-    try:
-        resp = await session.get(zhk_url, allow_redirects=True)
-        if resp.status_code != 200:
-            logger.warning("Cian newbuilding fetch %s → HTTP %d", zhk_url, resp.status_code)
-            return None
-        html = resp.text
 
     mfe = "newbuilding-card-desktop-fichering-frontend"
 
@@ -227,9 +213,6 @@ async def fetch_newbuilding(
         len(result.nested_offers),
     )
     return result
-    finally:
-        if close_session:
-            await session.close()
 
 
 # ---- private extractors ----

tradein-mvp/backend/tests/test_cian_newbuilding.py

@@ -1,4 +1,5 @@
 """Tests for cian_newbuilding scraper (Stage 6)."""
+
 from __future__ import annotations
 
 import pytest
@@ -247,42 +248,72 @@ def test_dataclass_defaults():
 
 
 # ---- async integration-style tests ----
+# Transport: fetch_newbuilding теперь использует BrowserFetcher (camoufox) вместо
+# curl_cffi AsyncSession (#972). Мокируем BrowserFetcher.fetch через monkeypatch.
+
+
+def _make_browser_fetcher_mock(html: str, monkeypatch: pytest.MonkeyPatch) -> None:
+    """Мокирует BrowserFetcher так, чтобы fetch() возвращал заданный html."""
+    from unittest.mock import AsyncMock, MagicMock
+
+    mock_fetcher = MagicMock()
+    mock_fetcher.fetch = AsyncMock(return_value=html)
+    mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
+    mock_fetcher.__aexit__ = AsyncMock(return_value=None)
+
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_newbuilding.BrowserFetcher",
+        lambda: mock_fetcher,
+    )
 
 
 @pytest.mark.asyncio
 async def test_fetch_newbuilding_returns_none_on_no_state(monkeypatch):
     """Если state extraction fails → return None gracefully."""
+    _make_browser_fetcher_mock("<html></html>", monkeypatch)
     monkeypatch.setattr(
         "app.services.scrapers.cian_newbuilding.extract_state",
         lambda html, mfe, key: None,
     )
-    from unittest.mock import AsyncMock, MagicMock
 
-    session = MagicMock()
+    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html></html>"
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
     assert result is None
 
 
 @pytest.mark.asyncio
-async def test_fetch_newbuilding_returns_none_on_http_error(monkeypatch):
+async def test_fetch_newbuilding_browser_error_propagates(monkeypatch):
-    """HTTP non-200 → return None gracefully."""
+    """BrowserFetcher.fetch raises → fetch_newbuilding propagates the exception."""
     from unittest.mock import AsyncMock, MagicMock
 
-    session = MagicMock()
+    mock_fetcher = MagicMock()
-    response = MagicMock()
+    mock_fetcher.fetch = AsyncMock(side_effect=RuntimeError("browser timeout"))
-    response.status_code = 403
+    mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
-    response.text = "Forbidden"
+    mock_fetcher.__aexit__ = AsyncMock(return_value=None)
-    session.get = AsyncMock(return_value=response)
+    monkeypatch.setattr(
-    session.close = AsyncMock()
+        "app.services.scrapers.cian_newbuilding.BrowserFetcher",
+        lambda: mock_fetcher,
+    )
 
-    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
+    with pytest.raises(RuntimeError, match="browser timeout"):
+        await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
+
+
+@pytest.mark.asyncio
+async def test_fetch_newbuilding_session_param_ignored(monkeypatch):
+    """session= param принимается для backward-compat, но не используется для page-fetch."""
+    _make_browser_fetcher_mock("<html></html>", monkeypatch)
+    monkeypatch.setattr(
+        "app.services.scrapers.cian_newbuilding.extract_state",
+        lambda html, mfe, key: None,
+    )
+
+    from unittest.mock import MagicMock
+
+    sentinel_session = MagicMock(name="should_not_be_called")
+    # Передаём session= — должно работать без ошибок, session.get НЕ вызывается
+    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=sentinel_session)
     assert result is None
+    sentinel_session.get.assert_not_called()
 
 
 @pytest.mark.asyncio
@@ -318,6 +349,7 @@ async def test_fetch_newbuilding_parses_state(monkeypatch):
         },
     }
 
+    _make_browser_fetcher_mock("<html>fake</html>", monkeypatch)
     monkeypatch.setattr(
         "app.services.scrapers.cian_newbuilding.extract_state",
         lambda html, mfe, key: fake_state,
@@ -327,16 +359,7 @@ async def test_fetch_newbuilding_parses_state(monkeypatch):
         lambda html: {},
     )
 
-    from unittest.mock import AsyncMock, MagicMock
+    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
-
-    session = MagicMock()
-    response = MagicMock()
-    response.status_code = 200
-    response.text = "<html>fake</html>"
-    session.get = AsyncMock(return_value=response)
-    session.close = AsyncMock()
-
-    result = await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", session=session)
 
     assert result is not None
     assert result.cian_internal_house_id == 54016