fix(scrapers): named BrowserSession concurrency const, fix Semaphore(3) doc drift (#1347) #1913

Merged
bot-backend merged 1 commit from fix/browser-semaphore-doc-1347 into main 2026-06-26 07:44:30 +00:00
2 changed files with 9 additions and 4 deletions

View file

@ -1142,7 +1142,7 @@ async def fetch_obj_checks(sess: BrowserSession, obj_id: int) -> tuple[Any, str]
# ── _fetch_*_safe wrappers for asyncio.gather in Phase B/C ───────────────────
# Каждый wrapper возвращает (kind, full_url, result_or_exception).
# Exceptions НЕ raise — помещаются в возвращаемый tuple.
# BrowserSession._sem (Semaphore(3)) bounds concurrency per-request автоматически.
# BrowserSession._sem (Semaphore(_BROWSER_CONCURRENCY)=8) bounds concurrency per-request.
async def _fetch_flats_safe(
@ -1881,7 +1881,7 @@ async def run_region_sweep(
# ── Phase B/C — per-object processing (resumable, parallel per-object) ─
# Все endpoint'ы одного obj_id запускаются параллельно через asyncio.gather.
# BrowserSession._sem (Semaphore(3)) ограничивает одновременные запросы.
# BrowserSession._sem (Semaphore(_BROWSER_CONCURRENCY)=8) ограничивает запросы.
# DB upserts выполняются последовательно после gather — один db Session
# не thread-safe для параллельной записи.
pdir = Path(photos_dir) if photos_dir else PHOTOS_DIR_DEFAULT

View file

@ -31,6 +31,10 @@ USER_AGENT = (
"(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
)
# Максимум одновременных in-page fetch() на одну BrowserSession;
# поднимали 3→6→8 под asyncio.gather fan-out, на наш.дом.рф ServicePipe WAF-tolerant.
_BROWSER_CONCURRENCY = 8
# Маппинг region_code → URL-сегмент города для реалистичного Referer.
# Не обязан быть исчерпывающим — fallback на /новостройки/строящиеся/.
REGION_LANDING_PATH = {
@ -79,7 +83,8 @@ class BrowserSession:
"""Keeps a live Playwright Chromium page; issues kn-API fetches inside it.
Use as async context manager. Concurrency is bounded by an asyncio.Semaphore
of size 3 (the same page can serve multiple concurrent fetches via JS-await).
of size ``_BROWSER_CONCURRENCY`` (the same page can serve multiple concurrent
fetches via JS-await).
"""
def __init__(
@ -104,7 +109,7 @@ class BrowserSession:
self._browser: Browser | None = None
self._context: BrowserContext | None = None
self._page: Page | None = None
self._sem = asyncio.Semaphore(8)
self._sem = asyncio.Semaphore(_BROWSER_CONCURRENCY)
self._request_count = 0
self._warmed_up = False