"""DOM.RF analytics scraper using Playwright (headless Chromium).

Bypasses:
  - ServicePipe WAF (Chromium executes JS challenge automatically)
  - Lazy-loading (wait for `networkidle` + page-specific triggers)
  - Cookie session (Chromium maintains it across navigations)

Captures: every JSON response from /api/* endpoints on the same domain.
Saves: data/raw/domrf_full/<section>/<endpoint>.json (one JSON per endpoint).

Usage:
    PYTHONIOENCODING=utf-8 python data/sql/30_scrape_domrf.py [--headed] [--page <name>]

Run --headed first to verify WAF passes; switch to default headless after.
"""
import asyncio, json, os, re, sys
from urllib.parse import urlsplit, parse_qs, urlencode, urlunsplit
from playwright.async_api import async_playwright

ROOT = os.path.join(os.path.dirname(__file__), '..', 'raw', 'domrf_full')
BASE = 'https://xn--80az8a.xn--d1aqf.xn--p1ai'

# 12 analytics pages — name (used as folder) → URL path
PAGES = [
    ('housing',          '/аналитика/показатели_жилищного_строительства'),
    ('launch',           '/аналитика/запуски-и-вводы?repYear=2026&calculationType=SQUARE&period=YEAR'),
    ('share_construction','/аналитика/долевое_строительство'),
    ('housing_dev',      '/аналитика/жилищное_строительство'),
    ('escrow',           '/аналитика/эскроу'),
    ('realization',      '/аналитика/реализация_строящихся_квартир'),
    ('sold_ready',       '/аналитика/распроданность-стройготовность?repYear=2026&repMonth=3'),
    ('quartirografia',   '/аналитика/квартирография'),
    ('commissioning',    '/аналитика/ввод_жилья'),
    ('mortgage_rates',   '/аналитика/ставки_предложений_по_ипотеке'),
    ('mortgage_stats',   '/аналитика/ипотечное_кредитование'),
    ('stat_series',      '/аналитика/статистические_ряды'),
]


def safe_filename(url: str) -> str:
    """Turn API URL into safe filename: path + query → snake_case."""
    parts = urlsplit(url)
    base = parts.path.split('/api/')[-1].strip('/')
    base = re.sub(r'[^A-Za-z0-9_\-]', '_', base)[:100]
    if parts.query:
        q = re.sub(r'[^A-Za-z0-9_\-]', '_', parts.query)[:80]
        base = f'{base}__{q}'
    return base + '.json'


def upgrade_size(url: str) -> str:
    """If URL has size=15 (or any small value), upgrade to size=1000 to get full data."""
    parts = urlsplit(url)
    if not parts.query:
        return url
    qs = parse_qs(parts.query, keep_blank_values=True)
    if 'size' in qs:
        qs['size'] = ['1000']
    new_q = urlencode({k: v[0] for k, v in qs.items()})
    return urlunsplit((parts.scheme, parts.netloc, parts.path, new_q, parts.fragment))


async def scrape_page(browser, name, path):
    out_dir = os.path.join(ROOT, name)
    os.makedirs(out_dir, exist_ok=True)
    print(f'\n=== {name} ({path}) ===')
    context = await browser.new_context(
        user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
                   '(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36',
        locale='ru-RU',
        viewport={'width': 1920, 'height': 1080},
    )
    page = await context.new_page()

    seen_urls = set()
    saved = []

    async def on_response(resp):
        url = resp.url
        # Match same-origin /api/ endpoints, skip yandex/sendsay trackers
        if 'xn--80az8a.xn--d1aqf.xn--p1ai' not in url or '/api/' not in url:
            return
        ct = (resp.headers.get('content-type') or '').lower()
        if 'json' not in ct:
            return
        if url in seen_urls:
            return
        seen_urls.add(url)
        try:
            text = await resp.text()
            fname = safe_filename(url)
            with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
                f.write(text)
            saved.append((fname, len(text), url))
            print(f'  saved {fname} ({len(text)}b)')
        except Exception as e:
            print(f'  ERR {url}: {e}')

    page.on('response', lambda r: asyncio.create_task(on_response(r)))

    full_url = BASE + path
    try:
        await page.goto(full_url, wait_until='domcontentloaded', timeout=60_000)
        # WAF challenge usually completes within 1-3 seconds
        try:
            await page.wait_for_load_state('networkidle', timeout=30_000)
        except:
            pass
        # Extra wait for lazy-loaded API calls
        await page.wait_for_timeout(8000)
        # Scroll multiple times to trigger any IntersectionObserver-bound fetches
        for _ in range(3):
            await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
            await page.wait_for_timeout(1500)
            await page.evaluate('window.scrollTo(0, document.body.scrollHeight / 2)')
            await page.wait_for_timeout(1500)
            await page.evaluate('window.scrollTo(0, 0)')
            await page.wait_for_timeout(1500)
        # Click first interactive button (triggers React state → API)
        try:
            await page.evaluate('''() => {
              const btn = document.querySelector('button:not([disabled])');
              if (btn) btn.click();
            }''')
            await page.wait_for_timeout(3000)
        except Exception:
            pass

        # Re-fetch with size=1000 for endpoints that have small page-size
        for url in list(seen_urls):
            up = upgrade_size(url)
            if up != url and up not in seen_urls:
                try:
                    resp = await page.evaluate(
                        '''(u) => fetch(u, {credentials:'include'})
                            .then(r => r.text()).then(t => ({status: 200, text: t}))''',
                        up,
                    )
                    if resp and resp.get('text'):
                        seen_urls.add(up)
                        fname = safe_filename(up)
                        with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
                            f.write(resp['text'])
                        saved.append((fname, len(resp['text']), up))
                        print(f'  upgraded {fname} ({len(resp["text"])}b)')
                except Exception as e:
                    print(f'  upgrade err {up}: {e}')
    finally:
        await context.close()
    return saved


async def main():
    headed = '--headed' in sys.argv
    target = None
    if '--page' in sys.argv:
        target = sys.argv[sys.argv.index('--page') + 1]

    pages = [(n, p) for n, p in PAGES if target is None or n == target]
    print(f'Scraping {len(pages)} pages, headed={headed}')
    os.makedirs(ROOT, exist_ok=True)

    async with async_playwright() as pw:
        browser = await pw.chromium.launch(headless=not headed)
        all_saved = {}
        for name, path in pages:
            try:
                all_saved[name] = await scrape_page(browser, name, path)
            except Exception as e:
                print(f'PAGE {name} FAILED: {e}')
                all_saved[name] = []
        await browser.close()

    print('\n=== SUMMARY ===')
    for name, saved in all_saved.items():
        total = sum(s[1] for s in saved)
        print(f'  {name:22s} files={len(saved):3d} total={total:>10}b')


if __name__ == '__main__':
    asyncio.run(main())