From 099b8e4b59adabf8b390eeab52440c44687a1aac Mon Sep 17 00:00:00 2001
From: lekss361 <lekss361@gendsgn.local>
Date: Sun, 24 May 2026 10:53:28 +0000
Subject: [PATCH] =?UTF-8?q?fix(scrapers):=20DOM.=D0=A0=D0=A4=20KN=20extras?=
 =?UTF-8?q?=20=E2=80=94=20bypass=20WAF=20+=20update=20renamed=20endpoints?=
 =?UTF-8?q?=20(#503)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .../services/scrapers/domrf_catalog_object.py |  5 +++
 backend/app/services/scrapers/domrf_kn.py     |  5 +++
 backend/app/services/scrapers/stealth.py      | 38 +++++++++++++++++++
 3 files changed, 48 insertions(+)

diff --git a/backend/app/services/scrapers/domrf_catalog_object.py b/backend/app/services/scrapers/domrf_catalog_object.py
index 1bc58d7d..3a8b765e 100644
--- a/backend/app/services/scrapers/domrf_catalog_object.py
+++ b/backend/app/services/scrapers/domrf_catalog_object.py
@@ -441,6 +441,11 @@ async def scrape_catalog_objects(
         # Страницы каталога публичные — Basic auth не нужен
         auth=None,
     ) as session:
+        # Warm-up: visit /сервисы/каталог-новостроек/ to obtain WAF cookies.
+        # DOM.РФ WAF (2026-05-24) блокирует SSR-страницы объектов без этих cookies.
+        # Idempotent — один вызов покрывает весь batch через этот BrowserSession.
+        await session.warm_up()
+
         for obj_id in obj_ids:
             stats["processed"] += 1
             ok = await scrape_catalog_object(db, session, obj_id, snapshot_date)
diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py
index 1c16fb50..765a93dc 100644
--- a/backend/app/services/scrapers/domrf_kn.py
+++ b/backend/app/services/scrapers/domrf_kn.py
@@ -1684,6 +1684,11 @@ async def run_region_sweep(
             load_state=load_state,
             save_state=save_state,
         ) as sess:
+            # Warm-up: visit /сервисы/каталог-новостроек/ to obtain WAF cookies
+            # (___dmpkit___, domain_sid). Required since 2026-05-24 — without these
+            # cookies all /сервисы/api/object/*/* requests return 403 + WAF HTML.
+            await sess.warm_up()
+
             # ── Phase A — fetch objects (skip on resume) ────────────────────
             if not resume_from_run_id:
                 for status in statuses:
diff --git a/backend/app/services/scrapers/stealth.py b/backend/app/services/scrapers/stealth.py
index 130e0903..acd590af 100644
--- a/backend/app/services/scrapers/stealth.py
+++ b/backend/app/services/scrapers/stealth.py
@@ -106,6 +106,7 @@ class BrowserSession:
         self._page: Page | None = None
         self._sem = asyncio.Semaphore(8)
         self._request_count = 0
+        self._warmed_up = False
 
     async def __aenter__(self) -> BrowserSession:
         await self._bootstrap()
@@ -142,6 +143,43 @@ class BrowserSession:
             logger.info("bootstrap: storage_state saved to %s", self.save_state)
         logger.info("bootstrap: page ready, WAF challenge passed")
 
+    async def warm_up(self, force: bool = False) -> None:
+        """Visit catalog listing to obtain WAF cookies (___dmpkit___, domain_sid).
+
+        DOM.РФ WAF (накатан между 2026-05-17 и 2026-05-24) требует session cookies для
+        любого запроса на /сервисы/api/object/*/*. Cookies выдаются JS-челленджем при
+        visit на /сервисы/каталог-новостроек/. Один warm-up на BrowserSession достаточен —
+        cookies валидны для всего домена.
+
+        Idempotent: повторные вызовы без force=True — no-op после первого успешного warm_up.
+        """
+        if self._warmed_up and not force:
+            return
+        if self._context is None:
+            raise RuntimeError("BrowserSession not bootstrapped — call inside `async with` block")
+        page = await self._context.new_page()
+        try:
+            await page.goto(
+                f"{BASE_URL}/сервисы/каталог-новостроек/",
+                wait_until="domcontentloaded",
+                timeout=30_000,
+            )
+            await page.wait_for_timeout(2000)  # JS challenge ставит cookies async
+            cookies = await self._context.cookies()
+            names = {c["name"] for c in cookies}
+            critical = {"___dmpkit___", "domain_sid"}
+            got = names & critical
+            if not got:
+                logger.warning(
+                    "warm_up: WAF cookies missing after catalog visit (have: %s)",
+                    sorted(names),
+                )
+            else:
+                logger.info("warm_up: got WAF cookies %s (total=%d)", sorted(got), len(cookies))
+            self._warmed_up = True
+        finally:
+            await page.close()
+
     async def get_json(self, path: str, params: dict[str, Any]) -> dict[str, Any]:
         """Fetch JSON via fetch() in browser. Retries with backoff on transient errors."""
         if self._page is None: