501 lines
20 KiB
Python
501 lines
20 KiB
Python
"""DOM.RF analytics scraper using Playwright (headless Chromium).
|
||
|
||
Bypasses:
|
||
- ServicePipe WAF (Chromium executes JS challenge automatically)
|
||
- Lazy-loading (wait for `networkidle` + page-specific triggers)
|
||
- Cookie session (Chromium maintains it across navigations)
|
||
|
||
Captures: every JSON response from /api/* endpoints on the same domain.
|
||
Saves: data/raw/domrf_full/<section>/<endpoint>.json (one JSON per endpoint).
|
||
|
||
Usage:
|
||
PYTHONIOENCODING=utf-8 python data/sql/30_scrape_domrf.py [--headed] [--page <name>]
|
||
|
||
Run --headed first to verify WAF passes; switch to default headless after.
|
||
"""
|
||
import asyncio, json, os, re, sys
|
||
from urllib.parse import urlsplit, parse_qs, urlencode, urlunsplit
|
||
from playwright.async_api import async_playwright
|
||
|
||
ROOT = os.path.join(os.path.dirname(__file__), '..', 'raw', 'domrf_full')
|
||
BASE = 'https://xn--80az8a.xn--d1aqf.xn--p1ai'
|
||
|
||
# 12 analytics pages — name (used as folder) → URL path
|
||
PAGES = [
|
||
('housing', '/аналитика/показатели_жилищного_строительства'),
|
||
('launch', '/аналитика/запуски-и-вводы?repYear=2026&calculationType=SQUARE&period=YEAR'),
|
||
('share_construction','/аналитика/долевое_строительство'),
|
||
('housing_dev', '/аналитика/жилищное_строительство'),
|
||
('escrow', '/аналитика/эскроу'),
|
||
('realization', '/аналитика/реализация_строящихся_квартир'),
|
||
('sold_ready', '/аналитика/распроданность-стройготовность?repYear=2026&repMonth=3'),
|
||
('quartirografia', '/аналитика/квартирография'),
|
||
('commissioning', '/аналитика/ввод_жилья'),
|
||
('mortgage_rates', '/аналитика/ставки_предложений_по_ипотеке'),
|
||
('mortgage_stats', '/аналитика/ипотечное_кредитование'),
|
||
('stat_series', '/аналитика/статистические_ряды'),
|
||
]
|
||
|
||
|
||
def safe_filename(url: str) -> str:
|
||
"""Turn API URL into safe filename: path + query → snake_case."""
|
||
parts = urlsplit(url)
|
||
base = parts.path.split('/api/')[-1].strip('/')
|
||
base = re.sub(r'[^A-Za-z0-9_\-]', '_', base)[:100]
|
||
if parts.query:
|
||
q = re.sub(r'[^A-Za-z0-9_\-]', '_', parts.query)[:80]
|
||
base = f'{base}__{q}'
|
||
return base + '.json'
|
||
|
||
|
||
def upgrade_size(url: str) -> str:
|
||
"""If URL has size=15 (or any small value), upgrade to size=1000 to get full data."""
|
||
parts = urlsplit(url)
|
||
if not parts.query:
|
||
return url
|
||
qs = parse_qs(parts.query, keep_blank_values=True)
|
||
if 'size' in qs:
|
||
qs['size'] = ['1000']
|
||
new_q = urlencode({k: v[0] for k, v in qs.items()})
|
||
return urlunsplit((parts.scheme, parts.netloc, parts.path, new_q, parts.fragment))
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Page-specific lazy-load triggers
|
||
# ---------------------------------------------------------------------------
|
||
# Most DOM.RF analytics pages render Material-UI tabs/selects; data fetches
|
||
# fire on tab/option switch. The "kitchen sink" trigger walks every tab and
|
||
# every Select option, with networkidle waits in between. Pages with custom
|
||
# UI (tree pickers, multi-step filters) get dedicated triggers below.
|
||
|
||
async def _wait_quiet(page, ms=2500):
|
||
try:
|
||
await page.wait_for_load_state('networkidle', timeout=8000)
|
||
except Exception:
|
||
pass
|
||
await page.wait_for_timeout(ms)
|
||
|
||
|
||
async def _click_all_tabs(page):
|
||
"""Click every Material-UI tab one-by-one, waiting between to capture XHR."""
|
||
tab_count = await page.evaluate('''() => document.querySelectorAll(
|
||
'[role="tab"], .MuiTab-root, button[data-tab]'
|
||
).length''')
|
||
print(f' tabs found: {tab_count}')
|
||
for i in range(tab_count):
|
||
try:
|
||
await page.evaluate(
|
||
'''(i) => {
|
||
const tabs = document.querySelectorAll(
|
||
'[role="tab"], .MuiTab-root, button[data-tab]'
|
||
);
|
||
if (tabs[i]) tabs[i].click();
|
||
}''',
|
||
i,
|
||
)
|
||
await _wait_quiet(page, 2000)
|
||
except Exception as e:
|
||
print(f' tab[{i}] err: {e}')
|
||
|
||
|
||
async def _iterate_selects(page, max_options=20):
|
||
"""Open every MUI Select / native <select>, pick each option, wait."""
|
||
select_count = await page.evaluate('''() => document.querySelectorAll(
|
||
'[role="combobox"], [role="button"][aria-haspopup="listbox"], select'
|
||
).length''')
|
||
print(f' selects found: {select_count}')
|
||
for i in range(select_count):
|
||
try:
|
||
opt_count = await page.evaluate(
|
||
'''async (i) => {
|
||
const sels = document.querySelectorAll(
|
||
'[role="combobox"], [role="button"][aria-haspopup="listbox"], select'
|
||
);
|
||
const sel = sels[i];
|
||
if (!sel) return 0;
|
||
if (sel.tagName === 'SELECT') return sel.options.length;
|
||
sel.click();
|
||
await new Promise(r => setTimeout(r, 800));
|
||
return document.querySelectorAll('[role="option"], li[data-value]').length;
|
||
}''',
|
||
i,
|
||
)
|
||
opt_count = min(opt_count, max_options)
|
||
for j in range(opt_count):
|
||
try:
|
||
await page.evaluate(
|
||
'''async ([i, j]) => {
|
||
const sels = document.querySelectorAll(
|
||
'[role="combobox"], [role="button"][aria-haspopup="listbox"], select'
|
||
);
|
||
const sel = sels[i];
|
||
if (!sel) return;
|
||
if (sel.tagName === 'SELECT') {
|
||
sel.selectedIndex = j;
|
||
sel.dispatchEvent(new Event('change', {bubbles: true}));
|
||
} else {
|
||
sel.click();
|
||
await new Promise(r => setTimeout(r, 600));
|
||
const opts = document.querySelectorAll('[role="option"], li[data-value]');
|
||
if (opts[j]) opts[j].click();
|
||
}
|
||
}''',
|
||
[i, j],
|
||
)
|
||
await _wait_quiet(page, 1500)
|
||
except Exception as e:
|
||
print(f' select[{i}].opt[{j}] err: {e}')
|
||
break
|
||
except Exception as e:
|
||
print(f' select[{i}] err: {e}')
|
||
|
||
|
||
async def _click_all_safe_buttons(page, deny=('Войти', 'Регистр', 'Выйти', 'Меню')):
|
||
"""Click every non-disabled button whose text doesn't match deny list."""
|
||
deny_js = json.dumps(list(deny))
|
||
btn_count = await page.evaluate(
|
||
'''(deny) => Array.from(document.querySelectorAll('button:not([disabled])'))
|
||
.filter(b => !deny.some(d => (b.textContent || '').includes(d))).length''',
|
||
deny,
|
||
)
|
||
print(f' safe buttons: {btn_count}')
|
||
for i in range(min(btn_count, 30)):
|
||
try:
|
||
await page.evaluate(
|
||
'''([i, deny]) => {
|
||
const bs = Array.from(document.querySelectorAll('button:not([disabled])'))
|
||
.filter(b => !deny.some(d => (b.textContent || '').includes(d)));
|
||
if (bs[i]) bs[i].click();
|
||
}''',
|
||
[i, list(deny)],
|
||
)
|
||
await _wait_quiet(page, 1500)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
async def trigger_kitchen_sink(page):
|
||
"""Generic: scroll, click all tabs, iterate all selects."""
|
||
for _ in range(2):
|
||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
|
||
await page.wait_for_timeout(1200)
|
||
await page.evaluate('window.scrollTo(0, 0)')
|
||
await page.wait_for_timeout(800)
|
||
await _click_all_tabs(page)
|
||
await _iterate_selects(page, max_options=10)
|
||
await _click_all_safe_buttons(page)
|
||
|
||
|
||
# ---------------------------------------------------------------------------
|
||
# Direct-API helpers — fetch from inside browser ctx so cookies/WAF survive
|
||
# ---------------------------------------------------------------------------
|
||
# These five pages don't fire XHR on user interaction — they're SSR-baked or
|
||
# expose a documented REST API. Instead of clicking we just call the APIs
|
||
# directly via page.evaluate(fetch()) and dump the JSON via the response hook.
|
||
# Page spec source: knowledge graph entities DomRF_Tab*_*_Apr26.
|
||
|
||
REGIONS_FULL_RU = list(range(1, 100)) # try region codes 1..99 (Rosstat-style)
|
||
|
||
|
||
async def _fetch_and_save(page, url, out_dir, seen_urls, saved):
|
||
"""Fetch URL inside browser context (so cookies/WAF stay valid), save body."""
|
||
if url in seen_urls:
|
||
return
|
||
try:
|
||
resp = await page.evaluate(
|
||
'''async (u) => {
|
||
const r = await fetch(u, {credentials: 'include'});
|
||
const t = await r.text();
|
||
return {status: r.status, ct: r.headers.get('content-type') || '', text: t};
|
||
}''',
|
||
url,
|
||
)
|
||
except Exception as e:
|
||
print(f' direct err {url}: {e}')
|
||
return
|
||
if not resp or resp.get('status', 0) >= 400:
|
||
return
|
||
text = resp.get('text') or ''
|
||
if not text or len(text) < 20:
|
||
return
|
||
seen_urls.add(url)
|
||
fname = safe_filename(url)
|
||
with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
|
||
f.write(text)
|
||
saved.append((fname, len(text), url))
|
||
print(f' direct {fname} ({len(text)}b)')
|
||
|
||
|
||
async def _dump_next_data(page, out_dir, saved):
|
||
"""Pull __NEXT_DATA__ JSON blob from the rendered HTML and save it."""
|
||
blob = await page.evaluate('''() => {
|
||
const el = document.getElementById('__NEXT_DATA__');
|
||
return el ? el.textContent : null;
|
||
}''')
|
||
if not blob:
|
||
return
|
||
fname = '__NEXT_DATA__.json'
|
||
with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
|
||
f.write(blob)
|
||
saved.append((fname, len(blob), 'inline:__NEXT_DATA__'))
|
||
print(f' ssr {fname} ({len(blob)}b)')
|
||
|
||
|
||
async def trigger_housing(page, out_dir, seen_urls, saved):
|
||
"""Tab 1: показатели жилищного строительства — SSR + portal-analytics REST."""
|
||
await _dump_next_data(page, out_dir, saved)
|
||
base = '/portal-analytics/api/dashboard'
|
||
# Aggregate (no filter) variants first
|
||
for ep in ('by-region', 'by-developer', 'by-room-count', 'by-flat-area'):
|
||
await _fetch_and_save(page, f'{base}/{ep}', out_dir, seen_urls, saved)
|
||
# Per-region drilldown
|
||
for rid in REGIONS_FULL_RU:
|
||
await _fetch_and_save(page, f'{base}/by-region?regionId={rid}', out_dir, seen_urls, saved)
|
||
await _fetch_and_save(page, f'{base}/by-room-count?regionId={rid}', out_dir, seen_urls, saved)
|
||
await _fetch_and_save(page, f'{base}/by-flat-area?regionId={rid}', out_dir, seen_urls, saved)
|
||
|
||
|
||
async def trigger_housing_dev(page, out_dir, seen_urls, saved):
|
||
"""Tab 2: жилищное строительство — different Redux slice, same portal-analytics base."""
|
||
await _dump_next_data(page, out_dir, saved)
|
||
# Probe common analytics endpoints
|
||
bases = [
|
||
'/portal-analytics/api/housing',
|
||
'/portal-analytics/api/dashboard',
|
||
'/аналитика/api/housing',
|
||
]
|
||
for b in bases:
|
||
for ep in ('summary', 'by-region', 'by-developer', 'by-period', 'overview', 'main'):
|
||
await _fetch_and_save(page, f'{b}/{ep}', out_dir, seen_urls, saved)
|
||
await trigger_kitchen_sink(page)
|
||
|
||
|
||
async def trigger_realization(page, out_dir, seen_urls, saved):
|
||
"""Tab 6: реализация — /аналитика/api/rpp/* documented in graph."""
|
||
await _dump_next_data(page, out_dir, saved)
|
||
base = '/аналитика/api/rpp'
|
||
rep_year, rep_month = 2026, 3
|
||
# RF-level (no regionCode)
|
||
for ep in ('total', 'housing', 'readyYear'):
|
||
await _fetch_and_save(
|
||
page, f'{base}/{ep}?repMonth={rep_month}&repYear={rep_year}',
|
||
out_dir, seen_urls, saved,
|
||
)
|
||
await _fetch_and_save(
|
||
page,
|
||
f'{base}/developer?typeSquare=total&repMonth={rep_month}&repYear={rep_year}'
|
||
f'&developerOrder=totalSquare:desc',
|
||
out_dir, seen_urls, saved,
|
||
)
|
||
# Per-region (89 регионов РФ — codes mostly in 1..99)
|
||
for rcode in REGIONS_FULL_RU:
|
||
for ep in ('total', 'housing', 'readyYear'):
|
||
await _fetch_and_save(
|
||
page,
|
||
f'{base}/{ep}?regionCode={rcode}&repMonth={rep_month}&repYear={rep_year}',
|
||
out_dir, seen_urls, saved,
|
||
)
|
||
for ts in ('total', 'living'):
|
||
await _fetch_and_save(
|
||
page,
|
||
f'{base}/developer?typeSquare={ts}®ionCode={rcode}'
|
||
f'&repMonth={rep_month}&repYear={rep_year}'
|
||
f'&developerOrder=totalSquare:desc',
|
||
out_dir, seen_urls, saved,
|
||
)
|
||
|
||
|
||
async def trigger_mortgage_rates(page, out_dir, seen_urls, saved):
|
||
"""Tab 10: ставки предложений по ипотеке — likely SSR; probe known APIs."""
|
||
await _dump_next_data(page, out_dir, saved)
|
||
bases = [
|
||
'/portal-analytics/api/mortgage-rates',
|
||
'/portal-analytics/api/mortgage',
|
||
'/аналитика/api/mortgage-rates',
|
||
'/аналитика/api/mortgage',
|
||
]
|
||
for b in bases:
|
||
for ep in ('', '/list', '/banks', '/offers', '/dashboard', '/summary'):
|
||
await _fetch_and_save(page, f'{b}{ep}', out_dir, seen_urls, saved)
|
||
await trigger_kitchen_sink(page)
|
||
|
||
|
||
async def trigger_stat_series(page, out_dir, seen_urls, saved):
|
||
"""Tab 12: статистические ряды — index of XLSX files baked into __NEXT_DATA__.
|
||
|
||
Per graph: data lives in props.initialState.housingData.list (CMS hierarchy).
|
||
XLSX URLs: /site/binaries/content/assets/domrf/xlsdashboard/{filename}.xlsx
|
||
Strategy: dump __NEXT_DATA__ + extract every .xlsx href and download.
|
||
"""
|
||
await _dump_next_data(page, out_dir, saved)
|
||
# Extract all .xlsx links from the rendered DOM and the __NEXT_DATA__ blob
|
||
links = await page.evaluate('''() => {
|
||
const set = new Set();
|
||
document.querySelectorAll('a[href]').forEach(a => {
|
||
const h = a.getAttribute('href');
|
||
if (h && h.toLowerCase().includes('.xlsx')) set.add(h);
|
||
});
|
||
const blob = document.getElementById('__NEXT_DATA__');
|
||
if (blob) {
|
||
const re = /["'](\\/site\\/binaries[^"']+\\.xlsx)["']/g;
|
||
let m;
|
||
while ((m = re.exec(blob.textContent)) !== null) set.add(m[1]);
|
||
}
|
||
return Array.from(set);
|
||
}''')
|
||
print(f' xlsx links found: {len(links)}')
|
||
xlsx_dir = os.path.join(out_dir, 'xlsx')
|
||
os.makedirs(xlsx_dir, exist_ok=True)
|
||
for href in links:
|
||
url = href if href.startswith('http') else BASE + href
|
||
# Download via fetch in browser context, then write bytes locally
|
||
try:
|
||
b64 = await page.evaluate(
|
||
'''async (u) => {
|
||
const r = await fetch(u, {credentials: 'include'});
|
||
if (!r.ok) return null;
|
||
const buf = await r.arrayBuffer();
|
||
let s = '';
|
||
const arr = new Uint8Array(buf);
|
||
for (let i = 0; i < arr.length; i++) s += String.fromCharCode(arr[i]);
|
||
return btoa(s);
|
||
}''',
|
||
url,
|
||
)
|
||
if not b64:
|
||
continue
|
||
import base64
|
||
data = base64.b64decode(b64)
|
||
fname = re.sub(r'[^A-Za-z0-9_.\-]', '_', os.path.basename(href.split('?')[0]))
|
||
with open(os.path.join(xlsx_dir, fname), 'wb') as f:
|
||
f.write(data)
|
||
saved.append((f'xlsx/{fname}', len(data), url))
|
||
print(f' xlsx {fname} ({len(data)}b)')
|
||
except Exception as e:
|
||
print(f' xlsx err {href}: {e}')
|
||
|
||
|
||
PAGE_TRIGGERS = {
|
||
'housing': trigger_housing,
|
||
'housing_dev': trigger_housing_dev,
|
||
'realization': trigger_realization,
|
||
'mortgage_rates': trigger_mortgage_rates,
|
||
'stat_series': trigger_stat_series,
|
||
}
|
||
|
||
|
||
async def scrape_page(browser, name, path):
|
||
out_dir = os.path.join(ROOT, name)
|
||
os.makedirs(out_dir, exist_ok=True)
|
||
print(f'\n=== {name} ({path}) ===')
|
||
context = await browser.new_context(
|
||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||
'(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36',
|
||
locale='ru-RU',
|
||
viewport={'width': 1920, 'height': 1080},
|
||
)
|
||
page = await context.new_page()
|
||
|
||
seen_urls = set()
|
||
saved = []
|
||
|
||
async def on_response(resp):
|
||
url = resp.url
|
||
# Match same-origin /api/ endpoints, skip yandex/sendsay trackers
|
||
if 'xn--80az8a.xn--d1aqf.xn--p1ai' not in url or '/api/' not in url:
|
||
return
|
||
ct = (resp.headers.get('content-type') or '').lower()
|
||
if 'json' not in ct:
|
||
return
|
||
if url in seen_urls:
|
||
return
|
||
seen_urls.add(url)
|
||
try:
|
||
text = await resp.text()
|
||
fname = safe_filename(url)
|
||
with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
|
||
f.write(text)
|
||
saved.append((fname, len(text), url))
|
||
print(f' saved {fname} ({len(text)}b)')
|
||
except Exception as e:
|
||
print(f' ERR {url}: {e}')
|
||
|
||
page.on('response', lambda r: asyncio.create_task(on_response(r)))
|
||
|
||
full_url = BASE + path
|
||
try:
|
||
await page.goto(full_url, wait_until='domcontentloaded', timeout=60_000)
|
||
try:
|
||
await page.wait_for_load_state('networkidle', timeout=30_000)
|
||
except Exception:
|
||
pass
|
||
await page.wait_for_timeout(5000)
|
||
|
||
# Per-page triggers (falls back to kitchen sink for everything new)
|
||
trigger = PAGE_TRIGGERS.get(name)
|
||
try:
|
||
if trigger is None:
|
||
await trigger_kitchen_sink(page)
|
||
else:
|
||
# New triggers take (page, out_dir, seen_urls, saved) so they
|
||
# can perform direct API fetches and dump SSR blobs themselves.
|
||
await trigger(page, out_dir, seen_urls, saved)
|
||
except Exception as e:
|
||
print(f' trigger err: {e}')
|
||
|
||
# Final settle
|
||
await _wait_quiet(page, 3000)
|
||
|
||
# Re-fetch with size=1000 for endpoints that have small page-size
|
||
for url in list(seen_urls):
|
||
up = upgrade_size(url)
|
||
if up != url and up not in seen_urls:
|
||
try:
|
||
resp = await page.evaluate(
|
||
'''(u) => fetch(u, {credentials:'include'})
|
||
.then(r => r.text()).then(t => ({status: 200, text: t}))''',
|
||
up,
|
||
)
|
||
if resp and resp.get('text'):
|
||
seen_urls.add(up)
|
||
fname = safe_filename(up)
|
||
with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
|
||
f.write(resp['text'])
|
||
saved.append((fname, len(resp['text']), up))
|
||
print(f' upgraded {fname} ({len(resp["text"])}b)')
|
||
except Exception as e:
|
||
print(f' upgrade err {up}: {e}')
|
||
finally:
|
||
await context.close()
|
||
return saved
|
||
|
||
|
||
async def main():
|
||
headed = '--headed' in sys.argv
|
||
target = None
|
||
if '--page' in sys.argv:
|
||
target = sys.argv[sys.argv.index('--page') + 1]
|
||
|
||
pages = [(n, p) for n, p in PAGES if target is None or n == target]
|
||
print(f'Scraping {len(pages)} pages, headed={headed}')
|
||
os.makedirs(ROOT, exist_ok=True)
|
||
|
||
async with async_playwright() as pw:
|
||
browser = await pw.chromium.launch(headless=not headed)
|
||
all_saved = {}
|
||
for name, path in pages:
|
||
try:
|
||
all_saved[name] = await scrape_page(browser, name, path)
|
||
except Exception as e:
|
||
print(f'PAGE {name} FAILED: {e}')
|
||
all_saved[name] = []
|
||
await browser.close()
|
||
|
||
print('\n=== SUMMARY ===')
|
||
for name, saved in all_saved.items():
|
||
total = sum(s[1] for s in saved)
|
||
print(f' {name:22s} files={len(saved):3d} total={total:>10}b')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
asyncio.run(main())
|