0f289c05b1
PR #322 used placeholder URLs: - /api/object/{id}/documents -> 404 (Playwright showed: 5 real endpoints: document/rpd / developer/report / project/documentation / documentation/other / document/permits). Replaced with 5 parallel fetches + merge into single upsert_documents call. - /api/object/{id}/checks -> 404. 6 checkboxes likely inline in bulk kn/object payload; endpoint /checks does not exist. Commented out _fetch_obj_checks_safe with TODO for separate investigation PR. - BrowserSession Semaphore(3) -> Semaphore(6) — double concurrency for intra-object gather. GlitchTip: kn_scrape_failures run #19 showed 40 failures each for both placeholder endpoints across first 30 objects of 1532.
201 lines
8.2 KiB
Python
201 lines
8.2 KiB
Python
"""Stealth-fetch utilities for наш.дом.рф scraping.
|
||
|
||
Pure-httpx hits ServicePipe WAF JS-challenge even with Playwright-exported
|
||
cookies (different TLS-fingerprint = blocked). The proven pattern from
|
||
data/sql/30_scrape_domrf.py is to issue `fetch()` calls *inside* the live
|
||
Chromium page — same JA3 fingerprint as a real browser, cookies/CSP/CORS
|
||
all handled natively.
|
||
|
||
BrowserSession keeps the browser+page open for the lifetime of a sweep and
|
||
exposes get_json() that calls page.evaluate(fetch_in_browser).
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import asyncio
|
||
import json
|
||
import logging
|
||
import random
|
||
from typing import Any
|
||
from urllib.parse import urlencode
|
||
|
||
from playwright.async_api import Browser, BrowserContext, Page, async_playwright
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
BASE_URL = "https://xn--80az8a.xn--d1aqf.xn--p1ai"
|
||
|
||
# Real Chrome 147 fingerprint (Win10 x64). Keep in sync with data/sql/30_scrape_domrf.py.
|
||
USER_AGENT = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||
"(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36"
|
||
)
|
||
|
||
# Маппинг region_code → URL-сегмент города для реалистичного Referer.
|
||
# Не обязан быть исчерпывающим — fallback на /новостройки/строящиеся/.
|
||
REGION_LANDING_PATH = {
|
||
66: "/новостройки/строящиеся/екатеринбург/",
|
||
77: "/новостройки/строящиеся/москва/",
|
||
78: "/новостройки/строящиеся/санкт-петербург/",
|
||
50: "/новостройки/строящиеся/московская-область/",
|
||
47: "/новостройки/строящиеся/ленинградская-область/",
|
||
}
|
||
|
||
|
||
def make_referer(region_code: int) -> str:
|
||
path = REGION_LANDING_PATH.get(region_code, "/новостройки/строящиеся/")
|
||
return BASE_URL + path
|
||
|
||
|
||
async def jitter_sleep(min_ms: int = 600, max_ms: int = 1500) -> None:
|
||
delay = random.uniform(min_ms / 1000, max_ms / 1000)
|
||
await asyncio.sleep(delay)
|
||
|
||
|
||
# JS executed inside the live page. Returns {ok, status, body, contentType}.
|
||
# Auth header is hardcoded `Basic MTpxd2U=` (decodes to `1:qwe`, the public
|
||
# debug auth shipped in the site's frontend bundle for the kn API).
|
||
_FETCH_JS = """
|
||
async ({url, auth}) => {
|
||
const headers = {'Accept': 'application/json, text/plain, */*'};
|
||
if (auth) headers['Authorization'] = auth;
|
||
try {
|
||
const r = await fetch(url, {credentials: 'include', headers});
|
||
const ctype = r.headers.get('content-type') || '';
|
||
const body = await r.text();
|
||
return {ok: r.ok, status: r.status, body, contentType: ctype};
|
||
} catch (e) {
|
||
return {ok: false, status: 0, body: String(e), contentType: ''};
|
||
}
|
||
}
|
||
"""
|
||
|
||
|
||
class WafBlockedError(RuntimeError):
|
||
"""Server returned non-JSON (likely ServicePipe JS-challenge)."""
|
||
|
||
|
||
class BrowserSession:
|
||
"""Keeps a live Playwright Chromium page; issues kn-API fetches inside it.
|
||
|
||
Use as async context manager. Concurrency is bounded by an asyncio.Semaphore
|
||
of size 3 (the same page can serve multiple concurrent fetches via JS-await).
|
||
"""
|
||
|
||
def __init__(
|
||
self,
|
||
region_code: int = 66,
|
||
headed: bool = False,
|
||
auth: str | None = "Basic MTpxd2U=",
|
||
load_state: str | None = None,
|
||
save_state: str | None = None,
|
||
) -> None:
|
||
self.region_code = region_code
|
||
self.headed = headed
|
||
self.auth = auth
|
||
# Path to a Playwright storage_state JSON (cookies + localStorage).
|
||
# If load_state is set, the context is created with these cookies preloaded —
|
||
# ServicePipe sees a returning user, no fresh JS-challenge.
|
||
# If save_state is set, the post-bootstrap state is dumped to that path
|
||
# so it can be committed and reused on a server.
|
||
self.load_state = load_state
|
||
self.save_state = save_state
|
||
self._pw: Any = None
|
||
self._browser: Browser | None = None
|
||
self._context: BrowserContext | None = None
|
||
self._page: Page | None = None
|
||
self._sem = asyncio.Semaphore(6)
|
||
self._request_count = 0
|
||
|
||
async def __aenter__(self) -> BrowserSession:
|
||
await self._bootstrap()
|
||
return self
|
||
|
||
async def __aexit__(self, *exc: Any) -> None:
|
||
if self._browser is not None:
|
||
await self._browser.close()
|
||
if self._pw is not None:
|
||
await self._pw.stop()
|
||
|
||
async def _bootstrap(self) -> None:
|
||
landing_url = make_referer(self.region_code)
|
||
logger.info("bootstrap: opening %s", landing_url)
|
||
self._pw = await async_playwright().start()
|
||
self._browser = await self._pw.chromium.launch(headless=not self.headed)
|
||
ctx_kwargs: dict[str, Any] = {
|
||
"user_agent": USER_AGENT,
|
||
"locale": "ru-RU",
|
||
"viewport": {"width": 1920, "height": 1080},
|
||
}
|
||
if self.load_state:
|
||
ctx_kwargs["storage_state"] = self.load_state
|
||
logger.info("bootstrap: loading saved storage_state from %s", self.load_state)
|
||
self._context = await self._browser.new_context(**ctx_kwargs)
|
||
self._page = await self._context.new_page()
|
||
await self._page.goto(landing_url, wait_until="domcontentloaded", timeout=30_000)
|
||
try:
|
||
await self._page.wait_for_load_state("networkidle", timeout=10_000)
|
||
except Exception:
|
||
pass
|
||
if self.save_state:
|
||
await self._context.storage_state(path=self.save_state)
|
||
logger.info("bootstrap: storage_state saved to %s", self.save_state)
|
||
logger.info("bootstrap: page ready, WAF challenge passed")
|
||
|
||
async def get_json(self, path: str, params: dict[str, Any]) -> dict[str, Any]:
|
||
"""Fetch JSON via fetch() in browser. Retries with backoff on transient errors."""
|
||
if self._page is None:
|
||
raise RuntimeError("BrowserSession not bootstrapped")
|
||
url = BASE_URL + path + "?" + urlencode(params)
|
||
last_err: Exception | None = None
|
||
for attempt in range(5):
|
||
async with self._sem:
|
||
await jitter_sleep()
|
||
try:
|
||
self._request_count += 1
|
||
result = await self._page.evaluate(_FETCH_JS, {"url": url, "auth": self.auth})
|
||
except Exception as e:
|
||
last_err = e
|
||
logger.warning("evaluate err attempt=%d: %r", attempt, e)
|
||
await asyncio.sleep(2**attempt)
|
||
continue
|
||
status = result.get("status", 0)
|
||
body = result.get("body", "")
|
||
ctype = result.get("contentType", "")
|
||
if status in (429,) or status >= 500 or status == 0:
|
||
last_err = RuntimeError(f"transient status={status}")
|
||
logger.warning("transient status=%d attempt=%d, backing off", status, attempt)
|
||
await asyncio.sleep(2**attempt)
|
||
continue
|
||
if "application/json" not in ctype:
|
||
# WAF returned the JS challenge HTML; re-bootstrap could help, but for now fail.
|
||
raise WafBlockedError(
|
||
f"non-JSON response: status={status} ctype={ctype} body[:120]={body[:120]!r}"
|
||
)
|
||
if status != 200:
|
||
raise RuntimeError(f"http {status}: {body[:200]}")
|
||
return json.loads(body)
|
||
raise RuntimeError(f"max retries exhausted: {last_err!r}")
|
||
|
||
@property
|
||
def request_count(self) -> int:
|
||
return self._request_count
|
||
|
||
async def download_binary(self, url: str) -> bytes:
|
||
"""Download a binary asset (e.g. photo PNG) reusing browser cookies/auth.
|
||
|
||
Uses Playwright APIRequest which goes through the browser context — same
|
||
cookies, same TLS fingerprint as the page itself.
|
||
"""
|
||
if self._context is None:
|
||
raise RuntimeError("BrowserSession not bootstrapped")
|
||
await jitter_sleep(200, 500) # Lighter throttle for static assets.
|
||
self._request_count += 1
|
||
resp = await self._context.request.get(
|
||
url,
|
||
headers={"Authorization": self.auth} if self.auth else {},
|
||
)
|
||
if resp.status != 200:
|
||
body = await resp.text()
|
||
raise RuntimeError(f"binary http {resp.status}: {body[:200]}")
|
||
return await resp.body()
|