diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py index d018cb6e..afdd6938 100644 --- a/backend/app/services/scrapers/domrf_kn.py +++ b/backend/app/services/scrapers/domrf_kn.py @@ -1142,7 +1142,7 @@ async def fetch_obj_checks(sess: BrowserSession, obj_id: int) -> tuple[Any, str] # ── _fetch_*_safe wrappers for asyncio.gather in Phase B/C ─────────────────── # Каждый wrapper возвращает (kind, full_url, result_or_exception). # Exceptions НЕ raise — помещаются в возвращаемый tuple. -# BrowserSession._sem (Semaphore(3)) bounds concurrency per-request автоматически. +# BrowserSession._sem (Semaphore(_BROWSER_CONCURRENCY)=8) bounds concurrency per-request. async def _fetch_flats_safe( @@ -1881,7 +1881,7 @@ async def run_region_sweep( # ── Phase B/C — per-object processing (resumable, parallel per-object) ─ # Все endpoint'ы одного obj_id запускаются параллельно через asyncio.gather. - # BrowserSession._sem (Semaphore(3)) ограничивает одновременные запросы. + # BrowserSession._sem (Semaphore(_BROWSER_CONCURRENCY)=8) ограничивает запросы. # DB upserts выполняются последовательно после gather — один db Session # не thread-safe для параллельной записи. pdir = Path(photos_dir) if photos_dir else PHOTOS_DIR_DEFAULT diff --git a/backend/app/services/scrapers/stealth.py b/backend/app/services/scrapers/stealth.py index 53b1f635..b73112ce 100644 --- a/backend/app/services/scrapers/stealth.py +++ b/backend/app/services/scrapers/stealth.py @@ -31,6 +31,10 @@ USER_AGENT = ( "(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36" ) +# Максимум одновременных in-page fetch() на одну BrowserSession; +# поднимали 3→6→8 под asyncio.gather fan-out, на наш.дом.рф ServicePipe WAF-tolerant. +_BROWSER_CONCURRENCY = 8 + # Маппинг region_code → URL-сегмент города для реалистичного Referer. # Не обязан быть исчерпывающим — fallback на /новостройки/строящиеся/. REGION_LANDING_PATH = { @@ -79,7 +83,8 @@ class BrowserSession: """Keeps a live Playwright Chromium page; issues kn-API fetches inside it. Use as async context manager. Concurrency is bounded by an asyncio.Semaphore - of size 3 (the same page can serve multiple concurrent fetches via JS-await). + of size ``_BROWSER_CONCURRENCY`` (the same page can serve multiple concurrent + fetches via JS-await). """ def __init__( @@ -104,7 +109,7 @@ class BrowserSession: self._browser: Browser | None = None self._context: BrowserContext | None = None self._page: Page | None = None - self._sem = asyncio.Semaphore(8) + self._sem = asyncio.Semaphore(_BROWSER_CONCURRENCY) self._request_count = 0 self._warmed_up = False