Merge pull request 'refactor(tradein/scraper-kit): cian providers on shared _base.py (#2362)' (#2374) from feat/tradein-scraper-kit-f4b-cian into main

tradein-mvp/backend/tests/test_kit_serp_proxy_pool.py

@@ -10,8 +10,14 @@ proxy_provider + use_pool=config.use_proxy_pool_browser в BrowserFetcher.
     т.е. env-прокси браузера, поведение байт-в-байт как до фикса.
   - proxy_provider задан + флаг on → BrowserFetcher получает провайдер + use_pool=True.
 
-BrowserFetcher монки-патчится на recording-заглушку в модуле каждого скрапера, так что
+BrowserFetcher монки-патчится на recording-заглушку, так что реального HTTP к
-реального HTTP к tradein-browser не происходит.
+tradein-browser не происходит. Yandex SERP всё ещё конструирует BrowserFetcher
+напрямую в своём модуле — патчим `yandex_serp.BrowserFetcher`. Cian SERP
+(#2362, F4b) переведён на общую фабрику `scraper_kit.providers._base.
+build_browser_fetcher()` — конструктор теперь вызывается ИЗ _base.py, поэтому
+патчим `BrowserFetcher` там, а не в `cian_serp` (там больше нет своей ссылки на
+класс). Проверяемый инвариант (какие kwargs долетают до BrowserFetcher)
+не меняется — меняется только точка патча, отражающая реальное место вызова.
 """
 
 from __future__ import annotations
@@ -20,6 +26,7 @@ import types
 from typing import Any
 
 import pytest
+from scraper_kit.providers import _base as kit_base
 from scraper_kit.providers.cian import serp as cian_serp
 from scraper_kit.providers.yandex import serp as yandex_serp
 
@@ -136,7 +143,9 @@ async def test_yandex_serp_provider_without_flag_stays_dark(
 async def test_cian_serp_wires_pool_when_flag_on(monkeypatch: pytest.MonkeyPatch) -> None:
     """proxy_provider + флаг on → BrowserFetcher получает провайдер + use_pool=True."""
     rec_fetcher, calls = _make_recorder()
-    monkeypatch.setattr(cian_serp, "BrowserFetcher", rec_fetcher)
+    # #2362: CianScraper.__aenter__ теперь строит BrowserFetcher через
+    # _base.build_browser_fetcher() — патчим класс там, не в cian_serp.
+    monkeypatch.setattr(kit_base, "BrowserFetcher", rec_fetcher)
     provider = _FakeProxyProvider()
 
     scraper = cian_serp.CianScraper(_config(use_pool=True), proxy_provider=provider)
@@ -153,7 +162,7 @@ async def test_cian_serp_wires_pool_when_flag_on(monkeypatch: pytest.MonkeyPatch
 async def test_cian_serp_no_pool_by_default(monkeypatch: pytest.MonkeyPatch) -> None:
     """Без provider/флага → BrowserFetcher(proxy_provider=None, use_pool=False) — ship-dark."""
     rec_fetcher, calls = _make_recorder()
-    monkeypatch.setattr(cian_serp, "BrowserFetcher", rec_fetcher)
+    monkeypatch.setattr(kit_base, "BrowserFetcher", rec_fetcher)
 
     scraper = cian_serp.CianScraper(_config(use_pool=False))
     async with scraper:

tradein-mvp/backend/tests/test_scraper_kit_pricehistory_session_parity.py

@@ -178,6 +178,12 @@ def test_fetch_detail_own_session_proxy_wiring_parity(monkeypatch: Any) -> None:
     real behaviour regression risk (#806: datacenter-IP gets banned by Cian without a
     proxy). This proves config=RealScraperConfig() (our actual call site) constructs
     AsyncSession(proxies=...) identically to the legacy direct-settings path.
+
+    #2362 (F4b): kit's own-session branch now builds the session via
+    `scraper_kit.providers._base.build_curl_cffi_session()` instead of constructing
+    `AsyncSession` directly in `detail.py` — so the spy patches `AsyncSession` where
+    the constructor call now actually happens (`_base`), not in `detail` anymore.
+    The asserted kwargs (proxies=...) are unchanged.
     """
     monkeypatch.setattr(settings, "cian_proxy_url_env", "http://test-proxy.local:8080")
 
@@ -204,7 +210,7 @@ def test_fetch_detail_own_session_proxy_wiring_parity(monkeypatch: Any) -> None:
     with patch("app.services.scrapers.cian_detail.AsyncSession", _make_fake_session("legacy")):
         asyncio.run(legacy_fetch_detail(url))
 
-    with patch("scraper_kit.providers.cian.detail.AsyncSession", _make_fake_session("kit")):
+    with patch("scraper_kit.providers._base.AsyncSession", _make_fake_session("kit")):
         asyncio.run(kit_fetch_detail(url, config=RealScraperConfig()))
 
     assert captured["legacy"].get("proxies") == captured["kit"].get("proxies")

tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/detail.py

@@ -17,11 +17,11 @@ import logging
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
-from curl_cffi.requests import AsyncSession
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from scraper_kit.cian_state_parser import extract_all_states, extract_state
+from scraper_kit.providers._base import build_curl_cffi_session
 from scraper_kit.providers._proxy import curl_proxy_url
 from scraper_kit.repair_state_normalizer import (
     infer_repair_state_from_text,
@@ -30,6 +30,8 @@ from scraper_kit.repair_state_normalizer import (
 from scraper_kit.snapshot_writer import upsert_listing_snapshot
 
 if TYPE_CHECKING:
+    from curl_cffi.requests import AsyncSession
+
     from scraper_kit.browser_fetcher import BrowserFetcher
     from scraper_kit.contracts import ProxyProvider, ScraperConfig
 
@@ -115,11 +117,9 @@ async def fetch_detail(
         # Пусто → прямое подключение (dev/no-op). curl_proxy_url: mark_health + release на выходе.
         _env = config.cian_proxy_url if config is not None else None
         with curl_proxy_url(config, proxy_provider, "cian", env_fallback_url=_env) as _proxy_url:
-            _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
+            own_session = build_curl_cffi_session(
-            own_session = AsyncSession(
+                proxy_url=_proxy_url,
-                impersonate="chrome120",
                 timeout=25.0,
-                proxies=_proxies,
                 headers={
                     "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                     "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",

tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/newbuilding.py

@@ -22,12 +22,13 @@ import re
 from dataclasses import dataclass, field
 from typing import TYPE_CHECKING, Any
 
-from curl_cffi.requests import AsyncSession  # type: ignore[import-untyped]
-
 from scraper_kit.browser_fetcher import BrowserFetcher
 from scraper_kit.cian_state_parser import extract_all_states, extract_state
+from scraper_kit.providers._base import build_curl_cffi_session
 
 if TYPE_CHECKING:
+    from curl_cffi.requests import AsyncSession  # type: ignore[import-untyped]
+
     from scraper_kit.contracts import ScraperConfig
 
 logger = logging.getLogger(__name__)
@@ -143,6 +144,15 @@ async def fetch_newbuilding(
 
     Returns: NewbuildingEnrichment, or None если fetch / parse failed.
     """
+    # #2362 (F4b): НЕ переведено на _base.build_browser_fetcher() намеренно.
+    # build_browser_fetcher() требует ScraperConfig mandatory (без default) — это
+    # весь смысл фабрики (footgun-класс #2322/#2330). Но `config` здесь ЗАКОННО
+    # Optional: admin.py debug-роут и test_scraper_kit_newbuilding_endpoint.py
+    # (test_cian_fetch_newbuilding_endpoint_none_without_config) вызывают
+    # fetch_newbuilding() БЕЗ config= и ожидают graceful endpoint=None (не крash).
+    # Подмена на build_browser_fetcher(config, ...) уронила бы AttributeError на
+    # config=None вместо текущего degrade-to-None — реальная регрессия поведения,
+    # а не чистый call-site swap. Оставлено как есть; см. PR #2362 caveat notes.
     endpoint = config.browser_http_endpoint if config is not None else None
     async with BrowserFetcher(source="cian", endpoint=endpoint) as browser:
         html = await browser.fetch(zhk_url)
@@ -651,11 +661,9 @@ async def resolve_cian_zhk_url(
     if session is None:
         # Mobile proxy wiring (#806 follow-up): resolve ЖК URL через мобильный прокси.
         _proxy_url = config.cian_proxy_url if config is not None else None
-        _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
+        session = build_curl_cffi_session(
-        session = AsyncSession(
+            proxy_url=_proxy_url,
-            impersonate="chrome120",
             timeout=15.0,
-            proxies=_proxies,
             headers={
                 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                 "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",
@@ -757,11 +765,9 @@ async def resolve_cian_zhk_url_via_search(
         # Mobile proxy wiring (#806): Cian блокирует datacenter-IP. proxy=None → прямое
         # подключение (dev / proxy-down fallback — single fetches survive direct).
         _proxy_url = config.cian_proxy_url if config is not None else None
-        _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
+        session = build_curl_cffi_session(
-        session = AsyncSession(
+            proxy_url=_proxy_url,
-            impersonate="chrome120",
             timeout=30.0,
-            proxies=_proxies,
             headers={
                 "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
                 "Accept-Language": "ru-RU,ru;q=0.9,en;q=0.8",

tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/serp.py

@@ -35,17 +35,18 @@ from typing import TYPE_CHECKING, Any
 from urllib.parse import urlencode
 
 from scraper_kit.base import BaseScraper, ScrapedLot
-from scraper_kit.browser_fetcher import BrowserFetcher
 from scraper_kit.cian_state_parser import extract_state
 from scraper_kit.house_type_normalizer import normalize_house_type
 from scraper_kit.price_brackets import get_price_seed_brackets
 from scraper_kit.pricing import BisectionConfig, ProbeFailPolicy, ProbeResult, walk_price_range
+from scraper_kit.providers._base import build_browser_fetcher
 from scraper_kit.repair_state_normalizer import (
     infer_repair_state_from_text,
     normalize_repair_state,
 )
 
 if TYPE_CHECKING:
+    from scraper_kit.browser_fetcher import BrowserFetcher
     from scraper_kit.contracts import ProxyProvider, ScraperConfig
 
 logger = logging.getLogger(__name__)
@@ -139,11 +140,10 @@ class CianScraper(BaseScraper):
         # fingerprint через tradein-browser сервис (#1806). Сервер роутит /fetch на
         # отдельный браузер+мобильный прокси для cian. Прокси ротирует IP сам, поэтому
         # code-side warm-up cookies и changeip-ротация больше не нужны.
-        self._browser = BrowserFetcher(
+        self._browser = build_browser_fetcher(
-            source="cian",
+            self._config,
-            endpoint=self._config.browser_http_endpoint,
+            "cian",
             proxy_provider=self._proxy_provider,
-            use_pool=self._config.use_proxy_pool_browser,
         )
         await self._browser.__aenter__()
         return self

tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/cian/session.py

@@ -10,11 +10,11 @@ import json
 import logging
 from typing import TYPE_CHECKING, Any
 
-from curl_cffi.requests import AsyncSession
 from sqlalchemy import text
 from sqlalchemy.orm import Session
 
 from scraper_kit.cian_state_parser import extract_state
+from scraper_kit.providers._base import build_curl_cffi_session
 
 if TYPE_CHECKING:
     from scraper_kit.contracts import ScraperConfig
@@ -119,12 +119,10 @@ async def verify_session(
         # при валидных DMIR_AUTH cookies. Без прокси verify всегда вернёт 403.
         # Пусто (env не задан) → прямое подключение (dev/no-op).
         _proxy_url = config.cian_proxy_url if config is not None else None
-        _proxies = {"http": _proxy_url, "https": _proxy_url} if _proxy_url else None
+        async with build_curl_cffi_session(
-        async with AsyncSession(
+            proxy_url=_proxy_url,
-            impersonate="chrome120",
-            cookies=cookies,
             timeout=20.0,
-            proxies=_proxies,
+            cookies=cookies,
         ) as session:
             resp = await session.get(_VALUATION_TEST_URL, allow_redirects=True)
             status = resp.status_code