From b89e0c3dafd95f71e6c44dd8ab247fb58383b074 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 02:59:50 +0300 Subject: [PATCH] =?UTF-8?q?fix(tradein/scraper-kit):=20=D0=BF=D1=80=D0=BE?= =?UTF-8?q?=D0=B1=D1=80=D0=BE=D1=81=20endpoint=3D=20=D0=B2=20yandex/cian?= =?UTF-8?q?=20newbuilding=20BrowserFetcher=20(#2322)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test_scraper_kit_newbuilding_endpoint.py | 140 ++++++++++++++++++ .../scraper_kit/providers/cian/newbuilding.py | 7 +- .../providers/yandex/newbuilding.py | 35 ++++- 3 files changed, 177 insertions(+), 5 deletions(-) create mode 100644 tradein-mvp/backend/tests/test_scraper_kit_newbuilding_endpoint.py diff --git a/tradein-mvp/backend/tests/test_scraper_kit_newbuilding_endpoint.py b/tradein-mvp/backend/tests/test_scraper_kit_newbuilding_endpoint.py new file mode 100644 index 00000000..e0d41687 --- /dev/null +++ b/tradein-mvp/backend/tests/test_scraper_kit_newbuilding_endpoint.py @@ -0,0 +1,140 @@ +"""Regression test — scraper_kit yandex/cian newbuilding providers proxy `endpoint=` +into `BrowserFetcher` (issue #2322, epic #2277 Group F1). + +Bug: `scraper_kit/providers/yandex/newbuilding.py` (`YandexNewbuildingScraper.fetch_jk`, +`resolve_yandex_jk_slug`) and `scraper_kit/providers/cian/newbuilding.py` +(`fetch_newbuilding`) constructed `BrowserFetcher(source=...)` WITHOUT the mandatory +`endpoint=` kwarg (required, no default in `BrowserFetcher.__init__`) — `TypeError` on +every real call, and the functions did not even expose a `config` parameter so callers +could not fix this from outside via DI (mirrors the pattern already used by +`resolve_cian_zhk_url_via_search` / other providers, e.g. `cian/serp.py`, +`domclick/serp.py`, `avito/serp.py`, `yandex/serp.py`). + +These tests prove `config.browser_http_endpoint` is actually threaded through to the +`BrowserFetcher(...)` constructor call args — not merely that the function "doesn't +crash" on a happy path. `BrowserFetcher` itself is replaced by a spy so the constructor +call is directly inspectable. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock + +import pytest +from scraper_kit.providers.cian.newbuilding import fetch_newbuilding +from scraper_kit.providers.yandex.newbuilding import ( + YandexNewbuildingScraper, + resolve_yandex_jk_slug, +) + +_TEST_ENDPOINT = "http://test-tradein-browser:9009/fetch" + + +def _spy_browser_fetcher(html: str) -> MagicMock: + """BrowserFetcher class-replacement spy: records the constructor call args/kwargs + and returns an async-context-manager mock whose `.fetch()` resolves to `html`.""" + mock_fetcher = MagicMock() + mock_fetcher.fetch = AsyncMock(return_value=html) + mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher) + mock_fetcher.__aexit__ = AsyncMock(return_value=None) + return MagicMock(return_value=mock_fetcher) + + +# ── cian.newbuilding.fetch_newbuilding ─────────────────────────────────────── + + +@pytest.mark.asyncio +async def test_cian_fetch_newbuilding_passes_endpoint_from_config(monkeypatch): + """config.browser_http_endpoint должен попасть в BrowserFetcher(endpoint=...).""" + spy = _spy_browser_fetcher("") + monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.BrowserFetcher", spy) + + config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT) + await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", config=config) + + spy.assert_called_once() + _, kwargs = spy.call_args + assert kwargs.get("source") == "cian" + assert kwargs.get("endpoint") == _TEST_ENDPOINT + + +@pytest.mark.asyncio +async def test_cian_fetch_newbuilding_endpoint_none_without_config(monkeypatch): + """Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом — сигнатура + больше не роняет TypeError на отсутствующем обязательном параметре конструктора.""" + spy = _spy_browser_fetcher("") + monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.BrowserFetcher", spy) + + await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/") + + spy.assert_called_once() + _, kwargs = spy.call_args + assert kwargs.get("source") == "cian" + assert kwargs.get("endpoint") is None + + +# ── yandex.newbuilding.YandexNewbuildingScraper.fetch_jk ───────────────────── + + +@pytest.mark.asyncio +async def test_yandex_fetch_jk_passes_endpoint_from_config(monkeypatch): + """config передан в конструктор скрапера → endpoint пробрасывается в BrowserFetcher.""" + spy = _spy_browser_fetcher("") + monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy) + + config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT) + scraper = YandexNewbuildingScraper(config=config) + await scraper.fetch_jk(jk_slug="tatlin", jk_id="1592987") + + spy.assert_called_once() + _, kwargs = spy.call_args + assert kwargs.get("source") == "yandex" + assert kwargs.get("endpoint") == _TEST_ENDPOINT + + +@pytest.mark.asyncio +async def test_yandex_fetch_jk_endpoint_none_without_config(monkeypatch): + """Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом.""" + spy = _spy_browser_fetcher("") + monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy) + + scraper = YandexNewbuildingScraper() + await scraper.fetch_jk(jk_slug="tatlin", jk_id="1592987") + + spy.assert_called_once() + _, kwargs = spy.call_args + assert kwargs.get("source") == "yandex" + assert kwargs.get("endpoint") is None + + +# ── yandex.newbuilding.resolve_yandex_jk_slug ──────────────────────────────── + + +@pytest.mark.asyncio +async def test_resolve_yandex_jk_slug_passes_endpoint_from_config(monkeypatch): + """config= (новый kwarg-only параметр) должен пробрасывать endpoint в BrowserFetcher.""" + spy = _spy_browser_fetcher("") + monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy) + + config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT) + await resolve_yandex_jk_slug("1592987", config=config) + + spy.assert_called_once() + _, kwargs = spy.call_args + assert kwargs.get("source") == "yandex" + assert kwargs.get("endpoint") == _TEST_ENDPOINT + + +@pytest.mark.asyncio +async def test_resolve_yandex_jk_slug_endpoint_none_without_config(monkeypatch): + """Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом.""" + spy = _spy_browser_fetcher("") + monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy) + + await resolve_yandex_jk_slug("1592987") + + spy.assert_called_once() + _, kwargs = spy.call_args + assert kwargs.get("source") == "yandex" + assert kwargs.get("endpoint") is None diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/newbuilding.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/newbuilding.py index eb74c544..36f1dd8f 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/newbuilding.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/newbuilding.py @@ -123,6 +123,7 @@ class NewbuildingEnrichment: async def fetch_newbuilding( zhk_url: str, *, + config: ScraperConfig | None = None, session: AsyncSession | None = None, # kept for backward-compat; unused for page fetch ) -> NewbuildingEnrichment | None: """Fetch ЖК catalog page и extract все containers. @@ -133,13 +134,17 @@ async def fetch_newbuilding( Args: zhk_url: e.g. 'https://zhk-ekaterininskiy-park-ekb-i.cian.ru/' + config: ScraperConfig — источник browser_http_endpoint для BrowserFetcher + (#2322 fix: раньше не принимался вообще, BrowserFetcher(source="cian") + конструировался без endpoint= → TypeError на любом вызове). session: зарезервирован для обратной совместимости с вызывающими; больше не используется для fetch страницы ЖК. Resolve-функции (resolve_cian_zhk_url_via_search) по-прежнему используют curl_cffi. Returns: NewbuildingEnrichment, or None если fetch / parse failed. """ - async with BrowserFetcher(source="cian") as browser: + endpoint = config.browser_http_endpoint if config is not None else None + async with BrowserFetcher(source="cian", endpoint=endpoint) as browser: html = await browser.fetch(zhk_url) # Cian ЖК-карточка: MFE 'newbuilding-card-desktop-frontend', key 'initialState' diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/yandex/newbuilding.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/yandex/newbuilding.py index b36b0c94..052fabd5 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/yandex/newbuilding.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/yandex/newbuilding.py @@ -22,7 +22,7 @@ from __future__ import annotations import logging import re from collections.abc import Callable -from typing import Any +from typing import TYPE_CHECKING, Any from pydantic import BaseModel, Field from selectolax.parser import HTMLParser, Node @@ -35,6 +35,12 @@ from scraper_kit.yandex_helpers import ( parse_house_type, ) +if TYPE_CHECKING: + # TYPE_CHECKING-импорт (как в avito/imv.py, cian/newbuilding.py) — избегаем + # hard import cycle и лишней зависимости в офлайн-тестах парсинга, которым + # BrowserFetcher/config не нужны. + from scraper_kit.contracts import ScraperConfig + logger = logging.getLogger(__name__) @@ -147,12 +153,22 @@ class YandexNewbuildingScraper(BaseScraper): base_url = "https://realty.yandex.ru" request_delay_sec = 5.0 # class default; instance value injected via delay_provider - def __init__(self, *, delay_provider: Callable[[str], float] | None = None) -> None: + def __init__( + self, + *, + config: ScraperConfig | None = None, + delay_provider: Callable[[str], float] | None = None, + ) -> None: super().__init__() # Strangler-инжекция (#2133): провайдер задержки приходит снаружи вместо # прямого импорта app.services.scraper_settings.get_scraper_delay. if delay_provider is not None: self.request_delay_sec = delay_provider(self.name) + # #2322 fix: config пробрасывается в BrowserFetcher(endpoint=...) — раньше + # BrowserFetcher(source="yandex") конструировался БЕЗ обязательного endpoint= + # kwarg, что давало TypeError на любом вызове fetch_jk и не давало caller'у + # исправить это снаружи (сигнатура вообще не принимала config). + self._config = config async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override] raise NotImplementedError( @@ -168,8 +184,9 @@ class YandexNewbuildingScraper(BaseScraper): через tradein-browser (camoufox) контейнер — единственный рабочий путь (#974). """ url = f"{self.base_url}/{city}/kupit/novostrojka/{jk_slug}-{jk_id}/" + endpoint = self._config.browser_http_endpoint if self._config is not None else None try: - async with BrowserFetcher(source="yandex") as fetcher: + async with BrowserFetcher(source="yandex", endpoint=endpoint) as fetcher: html = await fetcher.fetch(url) except Exception: logger.exception("yandex nb browser fetch failed: %s", url) @@ -303,6 +320,8 @@ class YandexNewbuildingScraper(BaseScraper): async def resolve_yandex_jk_slug( jk_id: str, city: str = "ekaterinburg", + *, + config: ScraperConfig | None = None, ) -> str | None: """Найти Yandex Realty slug для ЖК по его ext_id (jk_id) через SERP. @@ -316,6 +335,13 @@ async def resolve_yandex_jk_slug( Caller несёт ответственность за anti-bot sleep (зеркало cian_newbuilding.py). BrowserFetcher обязателен — Yandex Realty JS/anti-bot, httpx/curl не работают. + Args: + jk_id: Yandex Realty ext_id ЖК. + city: город (по умолчанию ekaterinburg). + config: ScraperConfig — источник browser_http_endpoint для BrowserFetcher + (#2322 fix: раньше не принимался вообще, BrowserFetcher(source="yandex") + конструировался без endpoint= → TypeError на любом вызове). + Returns: slug (str без id-суффикса), или None при ошибке / не найден. """ @@ -323,8 +349,9 @@ async def resolve_yandex_jk_slug( # Альтернативный путь через Яндекс Поиск (web SERP) менее надёжен из-за # вариативности разметки. Прямой realty.yandex.ru SERP — стабильнее. serp_url = f"https://realty.yandex.ru/{city}/kupit/novostrojka/?siteId={jk_id}" + endpoint = config.browser_http_endpoint if config is not None else None try: - async with BrowserFetcher(source="yandex") as fetcher: + async with BrowserFetcher(source="yandex", endpoint=endpoint) as fetcher: html = await fetcher.fetch(serp_url) except Exception as exc: logger.warning("resolve_yandex_jk_slug jk_id=%s browser fetch failed: %s", jk_id, exc)