gendesign/data/sql/40_scrape_history.py

"""Historical scrape: 12 months of 2025 + Jan/Feb 2026 for realization +
sold_ready. The DOM.RF API accepts repYear/repMonth params and returns
historical state — already verified by probe.

Output: data/raw/domrf_history/<section>/<YYYY-MM>/<endpoint>.json

Usage:
    PYTHONIOENCODING=utf-8 python data/sql/40_scrape_history.py
"""
import asyncio, json, os, re, sys
from urllib.parse import urlsplit, parse_qs, urlencode, urlunsplit
from playwright.async_api import async_playwright

ROOT = os.path.join(os.path.dirname(__file__), '..', 'raw', 'domrf_history')
BASE = 'https://xn--80az8a.xn--d1aqf.xn--p1ai'

# (year, month) pairs to backfill. 2025 full year + 2026 Jan/Feb.
# 2026-03 already has snapshot_date=2026-04-27 in domrf_realization.
PERIODS = [(2025, m) for m in range(1, 13)] + [(2026, 1), (2026, 2)]

REGIONS = list(range(1, 100))  # Rosstat-style 1..99


def safe_filename(url):
    parts = urlsplit(url)
    base = parts.path.split('/api/')[-1].strip('/')
    base = re.sub(r'[^A-Za-z0-9_\-]', '_', base)[:120]
    if parts.query:
        q = re.sub(r'[^A-Za-z0-9_\-]', '_', parts.query)[:80]
        base = f'{base}__{q}'
    return base + '.json'


async def warm_up(page, url):
    """Goto a section page so Chromium passes WAF for that subdomain segment."""
    await page.goto(BASE + url, wait_until='domcontentloaded', timeout=60_000)
    try:
        await page.wait_for_load_state('networkidle', timeout=30_000)
    except Exception:
        pass
    await page.wait_for_timeout(5000)


async def fetch_save(page, url, out_dir, saved):
    full = url if url.startswith('http') else BASE + url
    try:
        resp = await page.evaluate(
            '''async (u) => {
              const r = await fetch(u, {credentials: 'include'});
              const t = await r.text();
              return {s: r.status, ct: r.headers.get('content-type')||'', t};
            }''',
            full,
        )
    except Exception as e:
        print(f'  ERR {full}: {e}')
        return False
    if not resp or resp.get('s', 0) != 200:
        return False
    text = resp.get('t') or ''
    if len(text) < 20:
        return False
    fname = safe_filename(full)
    with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
        f.write(text)
    saved.append((fname, len(text)))
    return True


async def scrape_realization_period(page, year, month):
    out_dir = os.path.join(ROOT, 'realization', f'{year}-{month:02d}')
    os.makedirs(out_dir, exist_ok=True)
    saved = []
    base = '/аналитика/api/rpp'
    # RF-level total (no regionCode)
    for ep in ('total', 'housing', 'readyYear'):
        await fetch_save(page, f'{base}/{ep}?repMonth={month}&repYear={year}', out_dir, saved)
    await fetch_save(
        page,
        f'{base}/developer?typeSquare=total&repMonth={month}&repYear={year}'
        f'&developerOrder=totalSquare:desc',
        out_dir, saved,
    )
    # Per-region
    for rcode in REGIONS:
        for ep in ('total', 'housing', 'readyYear'):
            await fetch_save(
                page,
                f'{base}/{ep}?regionCode={rcode}&repMonth={month}&repYear={year}',
                out_dir, saved,
            )
        for ts in ('total', 'living'):
            await fetch_save(
                page,
                f'{base}/developer?typeSquare={ts}&regionCode={rcode}'
                f'&repMonth={month}&repYear={year}&developerOrder=totalSquare:desc',
                out_dir, saved,
            )
    print(f'  realization {year}-{month:02d}: {len(saved)} files')
    return len(saved)


async def scrape_sold_ready_period(page, year, month):
    out_dir = os.path.join(ROOT, 'sold_ready', f'{year}-{month:02d}')
    os.makedirs(out_dir, exist_ok=True)
    saved = []
    base = '/portal-analytics/api/ready-construction'
    for ep_path in [
        f'/index?repYear={year}&repMonth={month}',
        f'/charts?repYear={year}&repMonth={month}',
        f'/filters?repYear={year}&repMonth={month}',
        f'/ready-year-charts?repYear={year}&repMonth={month}',
        '/dynamics?dynamicChartType=readyPercChart,soldPercChart,soldReadyPercChart,squareSum',
    ]:
        await fetch_save(page, base + ep_path, out_dir, saved)
    print(f'  sold_ready  {year}-{month:02d}: {len(saved)} files')
    return len(saved)


async def main():
    os.makedirs(ROOT, exist_ok=True)
    async with async_playwright() as pw:
        browser = await pw.chromium.launch(headless=True)
        ctx = await browser.new_context(
            user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) '
                       'AppleWebKit/537.36 (KHTML, like Gecko) '
                       'Chrome/147.0.0.0 Safari/537.36',
            locale='ru-RU',
            viewport={'width': 1920, 'height': 1080},
        )
        page = await ctx.new_page()

        # Warm up realization page first
        print('Warming up: realization page')
        await warm_up(page, '/аналитика/реализация_строящихся_квартир')

        # Realization for all periods (uses /аналитика/api/rpp/*)
        for year, month in PERIODS:
            await scrape_realization_period(page, year, month)

        # Switch to sold_ready warm-up
        print('Warming up: sold_ready page')
        await warm_up(page, '/аналитика/распроданность-стройготовность?repYear=2026&repMonth=3')
        for year, month in PERIODS:
            await scrape_sold_ready_period(page, year, month)

        await browser.close()


if __name__ == '__main__':
    asyncio.run(main())