fix(tradein/scraper-kit): проброс endpoint= в yandex/cian newbuilding BrowserFetcher (#2322)
This commit is contained in:
bot-backend
parent
c4201f7d2a
commit
b89e0c3daf
3 changed files with 177 additions and 5 deletions
|
|
@ -0,0 +1,140 @@
|
|||
"""Regression test — scraper_kit yandex/cian newbuilding providers proxy `endpoint=`
|
||||
into `BrowserFetcher` (issue #2322, epic #2277 Group F1).
|
||||
|
||||
Bug: `scraper_kit/providers/yandex/newbuilding.py` (`YandexNewbuildingScraper.fetch_jk`,
|
||||
`resolve_yandex_jk_slug`) and `scraper_kit/providers/cian/newbuilding.py`
|
||||
(`fetch_newbuilding`) constructed `BrowserFetcher(source=...)` WITHOUT the mandatory
|
||||
`endpoint=` kwarg (required, no default in `BrowserFetcher.__init__`) — `TypeError` on
|
||||
every real call, and the functions did not even expose a `config` parameter so callers
|
||||
could not fix this from outside via DI (mirrors the pattern already used by
|
||||
`resolve_cian_zhk_url_via_search` / other providers, e.g. `cian/serp.py`,
|
||||
`domclick/serp.py`, `avito/serp.py`, `yandex/serp.py`).
|
||||
|
||||
These tests prove `config.browser_http_endpoint` is actually threaded through to the
|
||||
`BrowserFetcher(...)` constructor call args — not merely that the function "doesn't
|
||||
crash" on a happy path. `BrowserFetcher` itself is replaced by a spy so the constructor
|
||||
call is directly inspectable.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, MagicMock
|
||||
|
||||
import pytest
|
||||
from scraper_kit.providers.cian.newbuilding import fetch_newbuilding
|
||||
from scraper_kit.providers.yandex.newbuilding import (
|
||||
YandexNewbuildingScraper,
|
||||
resolve_yandex_jk_slug,
|
||||
)
|
||||
|
||||
_TEST_ENDPOINT = "http://test-tradein-browser:9009/fetch"
|
||||
|
||||
|
||||
def _spy_browser_fetcher(html: str) -> MagicMock:
|
||||
"""BrowserFetcher class-replacement spy: records the constructor call args/kwargs
|
||||
and returns an async-context-manager mock whose `.fetch()` resolves to `html`."""
|
||||
mock_fetcher = MagicMock()
|
||||
mock_fetcher.fetch = AsyncMock(return_value=html)
|
||||
mock_fetcher.__aenter__ = AsyncMock(return_value=mock_fetcher)
|
||||
mock_fetcher.__aexit__ = AsyncMock(return_value=None)
|
||||
return MagicMock(return_value=mock_fetcher)
|
||||
|
||||
|
||||
# ── cian.newbuilding.fetch_newbuilding ───────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cian_fetch_newbuilding_passes_endpoint_from_config(monkeypatch):
|
||||
"""config.browser_http_endpoint должен попасть в BrowserFetcher(endpoint=...)."""
|
||||
spy = _spy_browser_fetcher("<html></html>")
|
||||
monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.BrowserFetcher", spy)
|
||||
|
||||
config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT)
|
||||
await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/", config=config)
|
||||
|
||||
spy.assert_called_once()
|
||||
_, kwargs = spy.call_args
|
||||
assert kwargs.get("source") == "cian"
|
||||
assert kwargs.get("endpoint") == _TEST_ENDPOINT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cian_fetch_newbuilding_endpoint_none_without_config(monkeypatch):
|
||||
"""Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом — сигнатура
|
||||
больше не роняет TypeError на отсутствующем обязательном параметре конструктора."""
|
||||
spy = _spy_browser_fetcher("<html></html>")
|
||||
monkeypatch.setattr("scraper_kit.providers.cian.newbuilding.BrowserFetcher", spy)
|
||||
|
||||
await fetch_newbuilding("https://zhk-test-ekb-i.cian.ru/")
|
||||
|
||||
spy.assert_called_once()
|
||||
_, kwargs = spy.call_args
|
||||
assert kwargs.get("source") == "cian"
|
||||
assert kwargs.get("endpoint") is None
|
||||
|
||||
|
||||
# ── yandex.newbuilding.YandexNewbuildingScraper.fetch_jk ─────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_yandex_fetch_jk_passes_endpoint_from_config(monkeypatch):
|
||||
"""config передан в конструктор скрапера → endpoint пробрасывается в BrowserFetcher."""
|
||||
spy = _spy_browser_fetcher("<html></html>")
|
||||
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
||||
|
||||
config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT)
|
||||
scraper = YandexNewbuildingScraper(config=config)
|
||||
await scraper.fetch_jk(jk_slug="tatlin", jk_id="1592987")
|
||||
|
||||
spy.assert_called_once()
|
||||
_, kwargs = spy.call_args
|
||||
assert kwargs.get("source") == "yandex"
|
||||
assert kwargs.get("endpoint") == _TEST_ENDPOINT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_yandex_fetch_jk_endpoint_none_without_config(monkeypatch):
|
||||
"""Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом."""
|
||||
spy = _spy_browser_fetcher("<html></html>")
|
||||
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
||||
|
||||
scraper = YandexNewbuildingScraper()
|
||||
await scraper.fetch_jk(jk_slug="tatlin", jk_id="1592987")
|
||||
|
||||
spy.assert_called_once()
|
||||
_, kwargs = spy.call_args
|
||||
assert kwargs.get("source") == "yandex"
|
||||
assert kwargs.get("endpoint") is None
|
||||
|
||||
|
||||
# ── yandex.newbuilding.resolve_yandex_jk_slug ────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_yandex_jk_slug_passes_endpoint_from_config(monkeypatch):
|
||||
"""config= (новый kwarg-only параметр) должен пробрасывать endpoint в BrowserFetcher."""
|
||||
spy = _spy_browser_fetcher("<html></html>")
|
||||
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
||||
|
||||
config = SimpleNamespace(browser_http_endpoint=_TEST_ENDPOINT)
|
||||
await resolve_yandex_jk_slug("1592987", config=config)
|
||||
|
||||
spy.assert_called_once()
|
||||
_, kwargs = spy.call_args
|
||||
assert kwargs.get("source") == "yandex"
|
||||
assert kwargs.get("endpoint") == _TEST_ENDPOINT
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_resolve_yandex_jk_slug_endpoint_none_without_config(monkeypatch):
|
||||
"""Без config= (backward-compat) endpoint=None передаётся явным kwarg'ом."""
|
||||
spy = _spy_browser_fetcher("<html></html>")
|
||||
monkeypatch.setattr("scraper_kit.providers.yandex.newbuilding.BrowserFetcher", spy)
|
||||
|
||||
await resolve_yandex_jk_slug("1592987")
|
||||
|
||||
spy.assert_called_once()
|
||||
_, kwargs = spy.call_args
|
||||
assert kwargs.get("source") == "yandex"
|
||||
assert kwargs.get("endpoint") is None
|
||||
|
|
@ -123,6 +123,7 @@ class NewbuildingEnrichment:
|
|||
async def fetch_newbuilding(
|
||||
zhk_url: str,
|
||||
*,
|
||||
config: ScraperConfig | None = None,
|
||||
session: AsyncSession | None = None, # kept for backward-compat; unused for page fetch
|
||||
) -> NewbuildingEnrichment | None:
|
||||
"""Fetch ЖК catalog page и extract все containers.
|
||||
|
|
@ -133,13 +134,17 @@ async def fetch_newbuilding(
|
|||
|
||||
Args:
|
||||
zhk_url: e.g. 'https://zhk-ekaterininskiy-park-ekb-i.cian.ru/'
|
||||
config: ScraperConfig — источник browser_http_endpoint для BrowserFetcher
|
||||
(#2322 fix: раньше не принимался вообще, BrowserFetcher(source="cian")
|
||||
конструировался без endpoint= → TypeError на любом вызове).
|
||||
session: зарезервирован для обратной совместимости с вызывающими;
|
||||
больше не используется для fetch страницы ЖК. Resolve-функции
|
||||
(resolve_cian_zhk_url_via_search) по-прежнему используют curl_cffi.
|
||||
|
||||
Returns: NewbuildingEnrichment, or None если fetch / parse failed.
|
||||
"""
|
||||
async with BrowserFetcher(source="cian") as browser:
|
||||
endpoint = config.browser_http_endpoint if config is not None else None
|
||||
async with BrowserFetcher(source="cian", endpoint=endpoint) as browser:
|
||||
html = await browser.fetch(zhk_url)
|
||||
|
||||
# Cian ЖК-карточка: MFE 'newbuilding-card-desktop-frontend', key 'initialState'
|
||||
|
|
|
|||
|
|
@ -22,7 +22,7 @@ from __future__ import annotations
|
|||
import logging
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
from selectolax.parser import HTMLParser, Node
|
||||
|
|
@ -35,6 +35,12 @@ from scraper_kit.yandex_helpers import (
|
|||
parse_house_type,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
# TYPE_CHECKING-импорт (как в avito/imv.py, cian/newbuilding.py) — избегаем
|
||||
# hard import cycle и лишней зависимости в офлайн-тестах парсинга, которым
|
||||
# BrowserFetcher/config не нужны.
|
||||
from scraper_kit.contracts import ScraperConfig
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
|
|
@ -147,12 +153,22 @@ class YandexNewbuildingScraper(BaseScraper):
|
|||
base_url = "https://realty.yandex.ru"
|
||||
request_delay_sec = 5.0 # class default; instance value injected via delay_provider
|
||||
|
||||
def __init__(self, *, delay_provider: Callable[[str], float] | None = None) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
config: ScraperConfig | None = None,
|
||||
delay_provider: Callable[[str], float] | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Strangler-инжекция (#2133): провайдер задержки приходит снаружи вместо
|
||||
# прямого импорта app.services.scraper_settings.get_scraper_delay.
|
||||
if delay_provider is not None:
|
||||
self.request_delay_sec = delay_provider(self.name)
|
||||
# #2322 fix: config пробрасывается в BrowserFetcher(endpoint=...) — раньше
|
||||
# BrowserFetcher(source="yandex") конструировался БЕЗ обязательного endpoint=
|
||||
# kwarg, что давало TypeError на любом вызове fetch_jk и не давало caller'у
|
||||
# исправить это снаружи (сигнатура вообще не принимала config).
|
||||
self._config = config
|
||||
|
||||
async def fetch_around(self, lat: float, lon: float, radius_m: int = 1000) -> list: # type: ignore[override]
|
||||
raise NotImplementedError(
|
||||
|
|
@ -168,8 +184,9 @@ class YandexNewbuildingScraper(BaseScraper):
|
|||
через tradein-browser (camoufox) контейнер — единственный рабочий путь (#974).
|
||||
"""
|
||||
url = f"{self.base_url}/{city}/kupit/novostrojka/{jk_slug}-{jk_id}/"
|
||||
endpoint = self._config.browser_http_endpoint if self._config is not None else None
|
||||
try:
|
||||
async with BrowserFetcher(source="yandex") as fetcher:
|
||||
async with BrowserFetcher(source="yandex", endpoint=endpoint) as fetcher:
|
||||
html = await fetcher.fetch(url)
|
||||
except Exception:
|
||||
logger.exception("yandex nb browser fetch failed: %s", url)
|
||||
|
|
@ -303,6 +320,8 @@ class YandexNewbuildingScraper(BaseScraper):
|
|||
async def resolve_yandex_jk_slug(
|
||||
jk_id: str,
|
||||
city: str = "ekaterinburg",
|
||||
*,
|
||||
config: ScraperConfig | None = None,
|
||||
) -> str | None:
|
||||
"""Найти Yandex Realty slug для ЖК по его ext_id (jk_id) через SERP.
|
||||
|
||||
|
|
@ -316,6 +335,13 @@ async def resolve_yandex_jk_slug(
|
|||
Caller несёт ответственность за anti-bot sleep (зеркало cian_newbuilding.py).
|
||||
BrowserFetcher обязателен — Yandex Realty JS/anti-bot, httpx/curl не работают.
|
||||
|
||||
Args:
|
||||
jk_id: Yandex Realty ext_id ЖК.
|
||||
city: город (по умолчанию ekaterinburg).
|
||||
config: ScraperConfig — источник browser_http_endpoint для BrowserFetcher
|
||||
(#2322 fix: раньше не принимался вообще, BrowserFetcher(source="yandex")
|
||||
конструировался без endpoint= → TypeError на любом вызове).
|
||||
|
||||
Returns:
|
||||
slug (str без id-суффикса), или None при ошибке / не найден.
|
||||
"""
|
||||
|
|
@ -323,8 +349,9 @@ async def resolve_yandex_jk_slug(
|
|||
# Альтернативный путь через Яндекс Поиск (web SERP) менее надёжен из-за
|
||||
# вариативности разметки. Прямой realty.yandex.ru SERP — стабильнее.
|
||||
serp_url = f"https://realty.yandex.ru/{city}/kupit/novostrojka/?siteId={jk_id}"
|
||||
endpoint = config.browser_http_endpoint if config is not None else None
|
||||
try:
|
||||
async with BrowserFetcher(source="yandex") as fetcher:
|
||||
async with BrowserFetcher(source="yandex", endpoint=endpoint) as fetcher:
|
||||
html = await fetcher.fetch(serp_url)
|
||||
except Exception as exc:
|
||||
logger.warning("resolve_yandex_jk_slug jk_id=%s browser fetch failed: %s", jk_id, exc)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue