From debe55b8a0f85bc4c478d3beadf8200c04e1c602 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 4 Jul 2026 09:18:04 +0300 Subject: [PATCH] refactor(tradein/scraper-kit): domclick providers on shared _base.py (#2364) Group F4d (epic #2277, parent #2352). Migrated the ad-hoc BrowserFetcher construction in providers/domclick/serp.py::fetch_city to the shared build_browser_fetcher(config, source) helper from Foundation (#2358). Scope actually touched: - serp.py: replaced the local BrowserFetcher(source="domclick", endpoint=...) construction with build_browser_fetcher(self._config, "domclick"). - detail.py: NO changes needed. Grepped for BrowserFetcher(/AsyncSession(/ curl_cffi and found none - fetch_detail() receives an already-constructed browser_fetcher from the caller (orchestrator owns the session lifecycle, per its own module docstring), so there was nothing to migrate here. Caveat verified (same class as avito, per issue instructions): the old call site omitted use_pool, so it implicitly defaulted to False via the BrowserFetcher class default. build_browser_fetcher(config, source) reads use_pool=config.use_proxy_pool_browser instead - a real difference in the value passed in. However this call site does not pass proxy_provider (stays the default None), and BrowserFetcher._pool_proxy computes `use_pool and proxy_provider is not None` before ever using the pool - so with proxy_provider=None the effective behavior is identical regardless of what use_pool evaluates to (confirmed by reading browser_fetcher.py directly, matching the build_browser_fetcher docstring's own note that domclick is one of the providers not yet wired to the proxy pool, #2160 P4). No observable behavior change. Documented inline at the call site. Tests: domclick-specific suite (91 tests: golden parity, sweep, admin ingest parity, detail + detail kit parity) all green, plus cross-provider test_scraper_kit_pipeline_parity2.py (8 tests). Full backend suite: 3278 passed, 6 skipped, 1 pre-existing flake (test_search_cache_hit, #2208) - no new failures. --- .../src/scraper_kit/providers/domclick/serp.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/serp.py b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/serp.py index 890bb8a8..33ad905f 100644 --- a/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/serp.py +++ b/tradein-mvp/packages/scraper-kit/src/scraper_kit/providers/domclick/serp.py @@ -311,14 +311,20 @@ class DomClickScraper(BaseScraper): """ # TODO Layer B (#1846 follow-up): backfill renovation/wallType/priceHistory - from scraper_kit.browser_fetcher import BrowserFetcher + from scraper_kit.providers._base import build_browser_fetcher out_lots: list[ScrapedLot] = [] seen_ids: set[str] = set() - async with BrowserFetcher( - source="domclick", endpoint=self._config.browser_http_endpoint - ) as fetcher: + # build_browser_fetcher(config, source) без proxy_provider (дефолт None) — + # поведенчески идентично прежнему прямому BrowserFetcher(source=..., endpoint=...): + # use_pool теперь читает config.use_proxy_pool_browser (#2364 caveat, тот же + # класс что и avito) вместо хардкода False, но BrowserFetcher._pool_proxy + # вычисляет `use_pool and proxy_provider is not None` — при proxy_provider=None + # результат всегда False независимо от значения флага (см. _base.py + # build_browser_fetcher docstring: domclick пока не подключён к browser-proxy- + # пулу, #2160 P4 wiring gap). Поведение НЕ меняется. + async with build_browser_fetcher(self._config, "domclick") as fetcher: for bucket in ROOM_BUCKETS: logger.info( "domklik: BFF sweep rooms=%r city_id=%d pages_cap=%d",