Compare commits
No commits in common. "a77a175f66ef8ef10b0f5aaffaa157c3e0ad192b" and "b3e7a9f9fa1a2abe3522310de1f1fc9891bc2e3f" have entirely different histories.
a77a175f66
...
b3e7a9f9fa
1 changed files with 31 additions and 54 deletions
|
|
@ -6,8 +6,7 @@ Covers:
|
||||||
- None/empty → direct connection (no proxies dict)
|
- None/empty → direct connection (no proxies dict)
|
||||||
- _avito_proxies() helper returns correct dict shape
|
- _avito_proxies() helper returns correct dict shape
|
||||||
- per-scraper proxy wiring: YandexRealty, YandexValuation, CianValuation,
|
- per-scraper proxy wiring: YandexRealty, YandexValuation, CianValuation,
|
||||||
CianNewbuilding (fetch_newbuilding — now BrowserFetcher, proxy is browser-level),
|
CianNewbuilding (fetch_newbuilding + resolve_cian_zhk_url), AvitoIMV, AvitoDetail
|
||||||
AvitoIMV, AvitoDetail
|
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
@ -268,81 +267,59 @@ async def test_yandex_valuation_session_no_proxies_when_none():
|
||||||
|
|
||||||
|
|
||||||
# ── CianNewbuilding — fetch_newbuilding ───────────────────────────────────────
|
# ── CianNewbuilding — fetch_newbuilding ───────────────────────────────────────
|
||||||
# После #972 fetch_newbuilding тянет страницу ЖК через BrowserFetcher (camoufox),
|
|
||||||
# а НЕ через curl_cffi AsyncSession. Прокси применяются на уровне браузер-контейнера,
|
|
||||||
# а не через proxies= kwarg. AsyncSession для page-fetch больше НЕ создаётся.
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_cian_newbuilding_own_session_receives_proxies():
|
async def test_cian_newbuilding_own_session_receives_proxies():
|
||||||
"""fetch_newbuilding использует BrowserFetcher для page-fetch, не AsyncSession.
|
"""fetch_newbuilding creates own session with proxies= when no session passed."""
|
||||||
|
|
||||||
Проверяем:
|
|
||||||
- BrowserFetcher.fetch(url) вызывается с правильным URL
|
|
||||||
- curl_cffi AsyncSession НЕ создаётся для page-fetch (прокси — browser-level)
|
|
||||||
"""
|
|
||||||
from app.services.scrapers import cian_newbuilding
|
from app.services.scrapers import cian_newbuilding
|
||||||
|
|
||||||
zhk_url = "https://zhk-test.cian.ru/"
|
mock_session = MagicMock()
|
||||||
fetch_calls: list[str] = []
|
mock_resp = MagicMock()
|
||||||
session_ctor_calls: list[dict] = []
|
mock_resp.status_code = 200
|
||||||
|
mock_resp.text = "" # empty HTML → extract_state returns None → result None (ok)
|
||||||
|
mock_session.get = AsyncMock(return_value=mock_resp)
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
|
||||||
mock_fetcher = MagicMock()
|
captured_kwargs: dict = {}
|
||||||
mock_fetcher.fetch = AsyncMock(
|
|
||||||
side_effect=lambda url: fetch_calls.append(url) or "<html></html>"
|
|
||||||
)
|
|
||||||
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
|
|
||||||
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
|
|
||||||
|
|
||||||
def _fake_async_session(*args, **kwargs):
|
def _fake_session(*args, **kwargs):
|
||||||
session_ctor_calls.append(kwargs)
|
captured_kwargs.update(kwargs)
|
||||||
return MagicMock()
|
return mock_session
|
||||||
|
|
||||||
with patch.object(cian_newbuilding, "BrowserFetcher", lambda: mock_fetcher):
|
with patch.object(cian_newbuilding, "AsyncSession", _fake_session):
|
||||||
with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session):
|
with patch.object(cian_newbuilding, "settings", _mock_settings(_PROXY_URL)):
|
||||||
await cian_newbuilding.fetch_newbuilding(zhk_url)
|
await cian_newbuilding.fetch_newbuilding("https://zhk-test.cian.ru/")
|
||||||
|
|
||||||
# BrowserFetcher.fetch вызван с ЖК-url
|
assert captured_kwargs.get("proxies") == _EXPECTED_PROXIES
|
||||||
assert fetch_calls == [zhk_url]
|
|
||||||
# curl_cffi AsyncSession НЕ создавался для page-fetch
|
|
||||||
assert session_ctor_calls == []
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.asyncio
|
@pytest.mark.asyncio
|
||||||
async def test_cian_newbuilding_shared_session_not_recreated():
|
async def test_cian_newbuilding_shared_session_not_recreated():
|
||||||
"""fetch_newbuilding принимает session= для backward-compat, но использует BrowserFetcher.
|
"""fetch_newbuilding does NOT create own session when one is passed."""
|
||||||
|
|
||||||
Даже когда передан внешний session — страница ЖК тянется через BrowserFetcher,
|
|
||||||
AsyncSession не пересоздаётся, session.get не вызывается.
|
|
||||||
"""
|
|
||||||
from app.services.scrapers import cian_newbuilding
|
from app.services.scrapers import cian_newbuilding
|
||||||
|
|
||||||
zhk_url = "https://zhk-test.cian.ru/"
|
mock_session = MagicMock()
|
||||||
fetch_calls: list[str] = []
|
mock_resp = MagicMock()
|
||||||
session_ctor_calls: list[dict] = []
|
mock_resp.status_code = 200
|
||||||
|
mock_resp.text = ""
|
||||||
|
mock_session.get = AsyncMock(return_value=mock_resp)
|
||||||
|
mock_session.close = AsyncMock()
|
||||||
|
|
||||||
mock_fetcher = MagicMock()
|
session_ctor_calls = []
|
||||||
mock_fetcher.fetch = AsyncMock(
|
|
||||||
side_effect=lambda url: fetch_calls.append(url) or "<html></html>"
|
|
||||||
)
|
|
||||||
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
|
|
||||||
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
|
|
||||||
|
|
||||||
def _fake_async_session(*args, **kwargs):
|
def _fake_session(*args, **kwargs):
|
||||||
session_ctor_calls.append(kwargs)
|
session_ctor_calls.append(kwargs)
|
||||||
return MagicMock()
|
return MagicMock()
|
||||||
|
|
||||||
sentinel_session = MagicMock(name="external_session")
|
with patch.object(cian_newbuilding, "AsyncSession", _fake_session):
|
||||||
|
with patch.object(cian_newbuilding, "settings", _mock_settings(_PROXY_URL)):
|
||||||
|
await cian_newbuilding.fetch_newbuilding(
|
||||||
|
"https://zhk-test.cian.ru/", session=mock_session
|
||||||
|
)
|
||||||
|
|
||||||
with patch.object(cian_newbuilding, "BrowserFetcher", lambda: mock_fetcher):
|
# No own session should have been created (shared session was passed)
|
||||||
with patch.object(cian_newbuilding, "AsyncSession", _fake_async_session):
|
|
||||||
await cian_newbuilding.fetch_newbuilding(zhk_url, session=sentinel_session)
|
|
||||||
|
|
||||||
# BrowserFetcher.fetch вызван — именно через него тянется страница
|
|
||||||
assert fetch_calls == [zhk_url]
|
|
||||||
# Ни нового AsyncSession, ни вызова переданного session.get — proxy browser-level
|
|
||||||
assert session_ctor_calls == []
|
assert session_ctor_calls == []
|
||||||
sentinel_session.get.assert_not_called()
|
|
||||||
|
|
||||||
|
|
||||||
# ── AvitoDetail — fetch_detail own-session ───────────────────────────────────
|
# ── AvitoDetail — fetch_detail own-session ───────────────────────────────────
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue