fix(tradein): обновить test_scraper_proxy под BrowserFetcher-транспорт fetch_newbuilding #1542
1 changed files with 54 additions and 31 deletions
|
|
@ -6,7 +6,8 @@ Covers:
|
|||
- None/empty → direct connection (no proxies dict)
|
||||
- _avito_proxies() helper returns correct dict shape
|
||||
- per-scraper proxy wiring: YandexRealty, YandexValuation, CianValuation,
|
||||
CianNewbuilding (fetch_newbuilding + resolve_cian_zhk_url), AvitoIMV, AvitoDetail
|
||||
CianNewbuilding (fetch_newbuilding — now BrowserFetcher, proxy is browser-level),
|
||||
AvitoIMV, AvitoDetail
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
|
@ -267,59 +268,81 @@ async def test_yandex_valuation_session_no_proxies_when_none():
|
|||
|
||||
|
||||
# ── CianNewbuilding — fetch_newbuilding ───────────────────────────────────────
|
||||
# После #972 fetch_newbuilding тянет страницу ЖК через BrowserFetcher (camoufox),
|
||||
# а НЕ через curl_cffi AsyncSession. Прокси применяются на уровне браузер-контейнера,
|
||||
# а не через proxies= kwarg. AsyncSession для page-fetch больше НЕ создаётся.
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cian_newbuilding_own_session_receives_proxies():
|
||||
"""fetch_newbuilding creates own session with proxies= when no session passed."""
|
||||
"""fetch_newbuilding использует BrowserFetcher для page-fetch, не AsyncSession.
|
||||
|
||||
Проверяем:
|
||||
- BrowserFetcher.fetch(url) вызывается с правильным URL
|
||||
- curl_cffi AsyncSession НЕ создаётся для page-fetch (прокси — browser-level)
|
||||
"""
|
||||
from app.services.scrapers import cian_newbuilding
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = "" # empty HTML → extract_state returns None → result None (ok)
|
||||
mock_session.get = AsyncMock(return_value=mock_resp)
|
||||
mock_session.close = AsyncMock()
|
||||
zhk_url = "https://zhk-test.cian.ru/"
|
||||
fetch_calls: list[str] = []
|
||||
session_ctor_calls: list[dict] = []
|
||||
|
||||
captured_kwargs: dict = {}
|
||||
mock_fetcher = MagicMock()
|
||||
mock_fetcher.fetch = AsyncMock(
|
||||
side_effect=lambda url: fetch_calls.append(url) or "<html></html>"
|
||||
)
|
||||
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
|
||||
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
|
||||
|
||||
def _fake_session(*args, **kwargs):
|
||||
captured_kwargs.update(kwargs)
|
||||
return mock_session
|
||||
def _fake_async_session(*args, **kwargs):
|
||||
session_ctor_calls.append(kwargs)
|
||||
return MagicMock()
|
||||
|
||||
with patch.object(cian_newbuilding, "AsyncSession", _fake_session):
|
||||
with patch.object(cian_newbuilding, "settings", _mock_settings(_PROXY_URL)):
|
||||
await cian_newbuilding.fetch_newbuilding("https://zhk-test.cian.ru/")
|
||||
with patch.object(cian_newbuilding, "BrowserFetcher", lambda: mock_fetcher):
|
||||
with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session):
|
||||
await cian_newbuilding.fetch_newbuilding(zhk_url)
|
||||
|
||||
assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
|
||||
# BrowserFetcher.fetch вызван с ЖК-url
|
||||
assert fetch_calls == [zhk_url]
|
||||
# curl_cffi AsyncSession НЕ создавался для page-fetch
|
||||
assert session_ctor_calls == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cian_newbuilding_shared_session_not_recreated():
|
||||
"""fetch_newbuilding does NOT create own session when one is passed."""
|
||||
"""fetch_newbuilding принимает session= для backward-compat, но использует BrowserFetcher.
|
||||
|
||||
Даже когда передан внешний session — страница ЖК тянется через BrowserFetcher,
|
||||
AsyncSession не пересоздаётся, session.get не вызывается.
|
||||
"""
|
||||
from app.services.scrapers import cian_newbuilding
|
||||
|
||||
mock_session = MagicMock()
|
||||
mock_resp = MagicMock()
|
||||
mock_resp.status_code = 200
|
||||
mock_resp.text = ""
|
||||
mock_session.get = AsyncMock(return_value=mock_resp)
|
||||
mock_session.close = AsyncMock()
|
||||
zhk_url = "https://zhk-test.cian.ru/"
|
||||
fetch_calls: list[str] = []
|
||||
session_ctor_calls: list[dict] = []
|
||||
|
||||
session_ctor_calls = []
|
||||
mock_fetcher = MagicMock()
|
||||
mock_fetcher.fetch = AsyncMock(
|
||||
side_effect=lambda url: fetch_calls.append(url) or "<html></html>"
|
||||
)
|
||||
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
|
||||
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
|
||||
|
||||
def _fake_session(*args, **kwargs):
|
||||
def _fake_async_session(*args, **kwargs):
|
||||
session_ctor_calls.append(kwargs)
|
||||
return MagicMock()
|
||||
|
||||
with patch.object(cian_newbuilding, "AsyncSession", _fake_session):
|
||||
with patch.object(cian_newbuilding, "settings", _mock_settings(_PROXY_URL)):
|
||||
await cian_newbuilding.fetch_newbuilding(
|
||||
"https://zhk-test.cian.ru/", session=mock_session
|
||||
)
|
||||
sentinel_session = MagicMock(name="external_session")
|
||||
|
||||
# No own session should have been created (shared session was passed)
|
||||
with patch.object(cian_newbuilding, "BrowserFetcher", lambda: mock_fetcher):
|
||||
with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session):
|
||||
await cian_newbuilding.fetch_newbuilding(zhk_url, session=sentinel_session)
|
||||
|
||||
# BrowserFetcher.fetch вызван — именно через него тянется страница
|
||||
assert fetch_calls == [zhk_url]
|
||||
# Ни нового AsyncSession, ни вызова переданного session.get — proxy browser-level
|
||||
assert session_ctor_calls == []
|
||||
sentinel_session.get.assert_not_called()
|
||||
|
||||
|
||||
# ── AvitoDetail — fetch_detail own-session ───────────────────────────────────
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue