140 lines
6 KiB
Python
140 lines
6 KiB
Python
"""Regression test — scraper_kit yandex/cian newbuilding providers proxy `endpoint=`
|
|
into `BrowserFetcher` (issue #2322, epic #2277 Group F1).
|
|
|
|
Bug: `scraper_kit/providers/yandex/newbuilding.py` (`YandexNewbuildingScraper.fetch_jk`,
|
|
`resolve_yandex_jk_slug`) and `scraper_kit/providers/cian/newbuilding.py`
|
|
(`fetch_newbuilding`) constructed `BrowserFetcher(source=...)` WITHOUT the mandatory
|
|
`endpoint=` kwarg (required, no default in `BrowserFetcher.__init__`) — `TypeError` on
|
|
every real call, and the functions did not even expose a `config` parameter so callers
|
|
could not fix this from outside via DI (mirrors the pattern already used by
|
|
`resolve_cian_zhk_url_via_search` / other providers, e.g. `cian/serp.py`,
|
|
`domclick/serp.py`, `avito/serp.py`, `yandex/serp.py`).
|
|
|
|
These tests prove `config.browser_http_endpoint` is actually threaded through to the
|
|
`BrowserFetcher(...)` constructor call args — not merely that the function "doesn't
|
|
crash" on a happy path. `BrowserFetcher` itself is replaced by a spy so the constructor
|
|
call is directly inspectable.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
from unittest.mock import AsyncMock, MagicMock
|
|
|
|
import pytest
|
|
from scraper_kit.providers.cian.newbuilding import fetch_newbuilding
|
|
from scraper_kit.providers.yandex.newbuilding import (
|
|
YandexNewbuildingScraper,
|
|
resolve_yandex_jk_slug,
|
|
)
|
|
|
|
_TEST_ENDPOINT = "http://test-tradein-browser:9009/fetch"
|
|
|
|
|
|
def _spy_browser_fetcher(html: str) -> MagicMock:
|
|
"""BrowserFetcher class-replacement spy: records the constructor call args/kwargs
|
|
and returns an async-context-manager mock whose `.fetch()` resolves to `html`."""
|
|
mock_fetcher = MagicMock()
|
|
mock_fetcher.fetch = AsyncMock(return_value=html)
|
|
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
|
|
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
|
|
return MagicMock(return_value=mock_fetcher)
|
|
|
|
|
|
# ── cian.newbuilding.fetch_newbuilding ───────────────────────────────────────
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cian_fetch_newbuilding_passes_endpoint_from_config(monkeypatch):
|
|
"""config.browser_http_endpoint должен попасть в BrowserFetcher(endpoint=...)."""
|
|
spy = _spy_browser_fetcher("<html></html>")
|
|
monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.BrowserFetcher", spy)
|
|
|
|
config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT)
|
|
await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", config=config)
|
|
|
|
spy.assert_called_once()
|
|
_, kwargs = spy.call_args
|
|
assert kwargs.get("source") == "cian"
|
|
assert kwargs.get("endpoint") == _TEST_ENDPOINT
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cian_fetch_newbuilding_endpoint_none_without_config(monkeypatch):
|
|
"""Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом — сигнатура
|
|
больше не роняет TypeError на отсутствующем обязательном параметре конструктора."""
|
|
spy = _spy_browser_fetcher("<html></html>")
|
|
monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.BrowserFetcher", spy)
|
|
|
|
await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
|
|
|
|
spy.assert_called_once()
|
|
_, kwargs = spy.call_args
|
|
assert kwargs.get("source") == "cian"
|
|
assert kwargs.get("endpoint") is None
|
|
|
|
|
|
# ── yandex.newbuilding.YandexNewbuildingScraper.fetch_jk ─────────────────────
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_yandex_fetch_jk_passes_endpoint_from_config(monkeypatch):
|
|
"""config передан в конструктор скрапера → endpoint пробрасывается в BrowserFetcher."""
|
|
spy = _spy_browser_fetcher("<html></html>")
|
|
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
|
|
|
config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT)
|
|
scraper = YandexNewbuildingScraper(config=config)
|
|
await scraper.fetch_jk(jk_slug="tatlin", jk_id="1592987")
|
|
|
|
spy.assert_called_once()
|
|
_, kwargs = spy.call_args
|
|
assert kwargs.get("source") == "yandex"
|
|
assert kwargs.get("endpoint") == _TEST_ENDPOINT
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_yandex_fetch_jk_endpoint_none_without_config(monkeypatch):
|
|
"""Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом."""
|
|
spy = _spy_browser_fetcher("<html></html>")
|
|
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
|
|
|
scraper = YandexNewbuildingScraper()
|
|
await scraper.fetch_jk(jk_slug="tatlin", jk_id="1592987")
|
|
|
|
spy.assert_called_once()
|
|
_, kwargs = spy.call_args
|
|
assert kwargs.get("source") == "yandex"
|
|
assert kwargs.get("endpoint") is None
|
|
|
|
|
|
# ── yandex.newbuilding.resolve_yandex_jk_slug ────────────────────────────────
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_yandex_jk_slug_passes_endpoint_from_config(monkeypatch):
|
|
"""config= (новый kwarg-only параметр) должен пробрасывать endpoint в BrowserFetcher."""
|
|
spy = _spy_browser_fetcher("<html></html>")
|
|
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
|
|
|
config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT)
|
|
await resolve_yandex_jk_slug("1592987", config=config)
|
|
|
|
spy.assert_called_once()
|
|
_, kwargs = spy.call_args
|
|
assert kwargs.get("source") == "yandex"
|
|
assert kwargs.get("endpoint") == _TEST_ENDPOINT
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_resolve_yandex_jk_slug_endpoint_none_without_config(monkeypatch):
|
|
"""Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом."""
|
|
spy = _spy_browser_fetcher("<html></html>")
|
|
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
|
|
|
await resolve_yandex_jk_slug("1592987")
|
|
|
|
spy.assert_called_once()
|
|
_, kwargs = spy.call_args
|
|
assert kwargs.get("source") == "yandex"
|
|
assert kwargs.get("endpoint") is None
|