From 35bc792cf284598abc89d822e61836e65e207389 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Mon, 15 Jun 2026 19:02:13 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein):=20=D0=BE=D0=B1=D0=BD=D0=BE=D0=B2?= =?UTF-8?q?=D0=B8=D1=82=D1=8C=20test=5Fscraper=5Fproxy=20=D0=BF=D0=BE?= =?UTF-8?q?=D0=B4=20BrowserFetcher-=D1=82=D1=80=D0=B0=D0=BD=D1=81=D0=BF?= =?UTF-8?q?=D0=BE=D1=80=D1=82=20fetch=5Fnewbuilding=20=E2=80=94=20=D0=B7?= =?UTF-8?q?=D0=B5=D0=BB=D1=91=D0=BD=D1=8B=D0=B9=20CI-=D0=B3=D0=B5=D0=B9?= =?UTF-8?q?=D1=82=20(#972)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../backend/tests/test_scraper_proxy.py | 85 ++++++++++++------- 1 file changed, 54 insertions(+), 31 deletions(-) diff --git a/tradein-mvp/backend/tests/test_scraper_proxy.py b/tradein-mvp/backend/tests/test_scraper_proxy.py index 1cb22271..b124c305 100644 --- a/tradein-mvp/backend/tests/test_scraper_proxy.py +++ b/tradein-mvp/backend/tests/test_scraper_proxy.py @@ -6,7 +6,8 @@ Covers: - None/empty → direct connection (no proxies dict) - _avito_proxies() helper returns correct dict shape - per-scraper proxy wiring: YandexRealty, YandexValuation, CianValuation, - CianNewbuilding (fetch_newbuilding + resolve_cian_zhk_url), AvitoIMV, AvitoDetail + CianNewbuilding (fetch_newbuilding — now BrowserFetcher, proxy is browser-level), + AvitoIMV, AvitoDetail """ from __future__ import annotations @@ -267,59 +268,81 @@ async def test_yandex_valuation_session_no_proxies_when_none(): # ── CianNewbuilding — fetch_newbuilding ─────────────────────────────────────── +# После #972 fetch_newbuilding тянет страницу ЖК через BrowserFetcher (camoufox), +# а НЕ через curl_cffi AsyncSession. Прокси применяются на уровне браузер-контейнера, +# а не через proxies= kwarg. AsyncSession для page-fetch больше НЕ создаётся. @pytest.mark.asyncio async def test_cian_newbuilding_own_session_receives_proxies(): - """fetch_newbuilding creates own session with proxies= when no session passed.""" + """fetch_newbuilding использует BrowserFetcher для page-fetch, не AsyncSession. + + Проверяем: + - BrowserFetcher.fetch(url) вызывается с правильным URL + - curl_cffi AsyncSession НЕ создаётся для page-fetch (прокси — browser-level) + """ from app.services.scrapers import cian_newbuilding - mock_session = MagicMock() - mock_resp = MagicMock() - mock_resp.status_code = 200 - mock_resp.text = "" # empty HTML → extract_state returns None → result None (ok) - mock_session.get = AsyncMock(return_value=mock_resp) - mock_session.close = AsyncMock() + zhk_url = "https://zhk-test.cian.ru/" + fetch_calls: list[str] = [] + session_ctor_calls: list[dict] = [] - captured_kwargs: dict = {} + mock_fetcher = MagicMock() + mock_fetcher.fetch = AsyncMock( + side_effect=lambda url: fetch_calls.append(url) or "" + ) + mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher) + mock_fetcher.__aexit__ = AsyncMock(return_value=None) - def _fake_session(*args, **kwargs): - captured_kwargs.update(kwargs) - return mock_session + def _fake_async_session(*args, **kwargs): + session_ctor_calls.append(kwargs) + return MagicMock() - with patch.object(cian_newbuilding, "AsyncSession", _fake_session): - with patch.object(cian_newbuilding, "settings", _mock_settings(_PROXY_URL)): - await cian_newbuilding.fetch_newbuilding("https://zhk-test.cian.ru/") + with patch.object(cian_newbuilding, "BrowserFetcher", lambda: mock_fetcher): + with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session): + await cian_newbuilding.fetch_newbuilding(zhk_url) - assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES + # BrowserFetcher.fetch вызван с ЖК-url + assert fetch_calls == [zhk_url] + # curl_cffi AsyncSession НЕ создавался для page-fetch + assert session_ctor_calls == [] @pytest.mark.asyncio async def test_cian_newbuilding_shared_session_not_recreated(): - """fetch_newbuilding does NOT create own session when one is passed.""" + """fetch_newbuilding принимает session= для backward-compat, но использует BrowserFetcher. + + Даже когда передан внешний session — страница ЖК тянется через BrowserFetcher, + AsyncSession не пересоздаётся, session.get не вызывается. + """ from app.services.scrapers import cian_newbuilding - mock_session = MagicMock() - mock_resp = MagicMock() - mock_resp.status_code = 200 - mock_resp.text = "" - mock_session.get = AsyncMock(return_value=mock_resp) - mock_session.close = AsyncMock() + zhk_url = "https://zhk-test.cian.ru/" + fetch_calls: list[str] = [] + session_ctor_calls: list[dict] = [] - session_ctor_calls = [] + mock_fetcher = MagicMock() + mock_fetcher.fetch = AsyncMock( + side_effect=lambda url: fetch_calls.append(url) or "" + ) + mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher) + mock_fetcher.__aexit__ = AsyncMock(return_value=None) - def _fake_session(*args, **kwargs): + def _fake_async_session(*args, **kwargs): session_ctor_calls.append(kwargs) return MagicMock() - with patch.object(cian_newbuilding, "AsyncSession", _fake_session): - with patch.object(cian_newbuilding, "settings", _mock_settings(_PROXY_URL)): - await cian_newbuilding.fetch_newbuilding( - "https://zhk-test.cian.ru/", session=mock_session - ) + sentinel_session = MagicMock(name="external_session") - # No own session should have been created (shared session was passed) + with patch.object(cian_newbuilding, "BrowserFetcher", lambda: mock_fetcher): + with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session): + await cian_newbuilding.fetch_newbuilding(zhk_url, session=sentinel_session) + + # BrowserFetcher.fetch вызван — именно через него тянется страница + assert fetch_calls == [zhk_url] + # Ни нового AsyncSession, ни вызова переданного session.get — proxy browser-level assert session_ctor_calls == [] + sentinel_session.get.assert_not_called() # ── AvitoDetail — fetch_detail own-session ───────────────────────────────────