181 lines
7.4 KiB
Python
181 lines
7.4 KiB
Python
"""DOM.RF analytics scraper using Playwright (headless Chromium).
|
||
|
||
Bypasses:
|
||
- ServicePipe WAF (Chromium executes JS challenge automatically)
|
||
- Lazy-loading (wait for `networkidle` + page-specific triggers)
|
||
- Cookie session (Chromium maintains it across navigations)
|
||
|
||
Captures: every JSON response from /api/* endpoints on the same domain.
|
||
Saves: data/raw/domrf_full/<section>/<endpoint>.json (one JSON per endpoint).
|
||
|
||
Usage:
|
||
PYTHONIOENCODING=utf-8 python data/sql/30_scrape_domrf.py [--headed] [--page <name>]
|
||
|
||
Run --headed first to verify WAF passes; switch to default headless after.
|
||
"""
|
||
import asyncio, json, os, re, sys
|
||
from urllib.parse import urlsplit, parse_qs, urlencode, urlunsplit
|
||
from playwright.async_api import async_playwright
|
||
|
||
ROOT = os.path.join(os.path.dirname(__file__), '..', 'raw', 'domrf_full')
|
||
BASE = 'https://xn--80az8a.xn--d1aqf.xn--p1ai'
|
||
|
||
# 12 analytics pages — name (used as folder) → URL path
|
||
PAGES = [
|
||
('housing', '/аналитика/показатели_жилищного_строительства'),
|
||
('launch', '/аналитика/запуски-и-вводы?repYear=2026&calculationType=SQUARE&period=YEAR'),
|
||
('share_construction','/аналитика/долевое_строительство'),
|
||
('housing_dev', '/аналитика/жилищное_строительство'),
|
||
('escrow', '/аналитика/эскроу'),
|
||
('realization', '/аналитика/реализация_строящихся_квартир'),
|
||
('sold_ready', '/аналитика/распроданность-стройготовность?repYear=2026&repMonth=3'),
|
||
('quartirografia', '/аналитика/квартирография'),
|
||
('commissioning', '/аналитика/ввод_жилья'),
|
||
('mortgage_rates', '/аналитика/ставки_предложений_по_ипотеке'),
|
||
('mortgage_stats', '/аналитика/ипотечное_кредитование'),
|
||
('stat_series', '/аналитика/статистические_ряды'),
|
||
]
|
||
|
||
|
||
def safe_filename(url: str) -> str:
|
||
"""Turn API URL into safe filename: path + query → snake_case."""
|
||
parts = urlsplit(url)
|
||
base = parts.path.split('/api/')[-1].strip('/')
|
||
base = re.sub(r'[^A-Za-z0-9_\-]', '_', base)[:100]
|
||
if parts.query:
|
||
q = re.sub(r'[^A-Za-z0-9_\-]', '_', parts.query)[:80]
|
||
base = f'{base}__{q}'
|
||
return base + '.json'
|
||
|
||
|
||
def upgrade_size(url: str) -> str:
|
||
"""If URL has size=15 (or any small value), upgrade to size=1000 to get full data."""
|
||
parts = urlsplit(url)
|
||
if not parts.query:
|
||
return url
|
||
qs = parse_qs(parts.query, keep_blank_values=True)
|
||
if 'size' in qs:
|
||
qs['size'] = ['1000']
|
||
new_q = urlencode({k: v[0] for k, v in qs.items()})
|
||
return urlunsplit((parts.scheme, parts.netloc, parts.path, new_q, parts.fragment))
|
||
|
||
|
||
async def scrape_page(browser, name, path):
|
||
out_dir = os.path.join(ROOT, name)
|
||
os.makedirs(out_dir, exist_ok=True)
|
||
print(f'\n=== {name} ({path}) ===')
|
||
context = await browser.new_context(
|
||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||
'(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36',
|
||
locale='ru-RU',
|
||
viewport={'width': 1920, 'height': 1080},
|
||
)
|
||
page = await context.new_page()
|
||
|
||
seen_urls = set()
|
||
saved = []
|
||
|
||
async def on_response(resp):
|
||
url = resp.url
|
||
# Match same-origin /api/ endpoints, skip yandex/sendsay trackers
|
||
if 'xn--80az8a.xn--d1aqf.xn--p1ai' not in url or '/api/' not in url:
|
||
return
|
||
ct = (resp.headers.get('content-type') or '').lower()
|
||
if 'json' not in ct:
|
||
return
|
||
if url in seen_urls:
|
||
return
|
||
seen_urls.add(url)
|
||
try:
|
||
text = await resp.text()
|
||
fname = safe_filename(url)
|
||
with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
|
||
f.write(text)
|
||
saved.append((fname, len(text), url))
|
||
print(f' saved {fname} ({len(text)}b)')
|
||
except Exception as e:
|
||
print(f' ERR {url}: {e}')
|
||
|
||
page.on('response', lambda r: asyncio.create_task(on_response(r)))
|
||
|
||
full_url = BASE + path
|
||
try:
|
||
await page.goto(full_url, wait_until='domcontentloaded', timeout=60_000)
|
||
# WAF challenge usually completes within 1-3 seconds
|
||
try:
|
||
await page.wait_for_load_state('networkidle', timeout=30_000)
|
||
except:
|
||
pass
|
||
# Extra wait for lazy-loaded API calls
|
||
await page.wait_for_timeout(8000)
|
||
# Scroll multiple times to trigger any IntersectionObserver-bound fetches
|
||
for _ in range(3):
|
||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
|
||
await page.wait_for_timeout(1500)
|
||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight / 2)')
|
||
await page.wait_for_timeout(1500)
|
||
await page.evaluate('window.scrollTo(0, 0)')
|
||
await page.wait_for_timeout(1500)
|
||
# Click first interactive button (triggers React state → API)
|
||
try:
|
||
await page.evaluate('''() => {
|
||
const btn = document.querySelector('button:not([disabled])');
|
||
if (btn) btn.click();
|
||
}''')
|
||
await page.wait_for_timeout(3000)
|
||
except Exception:
|
||
pass
|
||
|
||
# Re-fetch with size=1000 for endpoints that have small page-size
|
||
for url in list(seen_urls):
|
||
up = upgrade_size(url)
|
||
if up != url and up not in seen_urls:
|
||
try:
|
||
resp = await page.evaluate(
|
||
'''(u) => fetch(u, {credentials:'include'})
|
||
.then(r => r.text()).then(t => ({status: 200, text: t}))''',
|
||
up,
|
||
)
|
||
if resp and resp.get('text'):
|
||
seen_urls.add(up)
|
||
fname = safe_filename(up)
|
||
with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
|
||
f.write(resp['text'])
|
||
saved.append((fname, len(resp['text']), up))
|
||
print(f' upgraded {fname} ({len(resp["text"])}b)')
|
||
except Exception as e:
|
||
print(f' upgrade err {up}: {e}')
|
||
finally:
|
||
await context.close()
|
||
return saved
|
||
|
||
|
||
async def main():
|
||
headed = '--headed' in sys.argv
|
||
target = None
|
||
if '--page' in sys.argv:
|
||
target = sys.argv[sys.argv.index('--page') + 1]
|
||
|
||
pages = [(n, p) for n, p in PAGES if target is None or n == target]
|
||
print(f'Scraping {len(pages)} pages, headed={headed}')
|
||
os.makedirs(ROOT, exist_ok=True)
|
||
|
||
async with async_playwright() as pw:
|
||
browser = await pw.chromium.launch(headless=not headed)
|
||
all_saved = {}
|
||
for name, path in pages:
|
||
try:
|
||
all_saved[name] = await scrape_page(browser, name, path)
|
||
except Exception as e:
|
||
print(f'PAGE {name} FAILED: {e}')
|
||
all_saved[name] = []
|
||
await browser.close()
|
||
|
||
print('\n=== SUMMARY ===')
|
||
for name, saved in all_saved.items():
|
||
total = sum(s[1] for s in saved)
|
||
print(f' {name:22s} files={len(saved):3d} total={total:>10}b')
|
||
|
||
|
||
if __name__ == '__main__':
|
||
asyncio.run(main())
|