"""DOM.RF analytics scraper using Playwright (headless Chromium).

Bypasses:
  - ServicePipe WAF (Chromium executes JS challenge automatically)
  - Lazy-loading (wait for `networkidle` + page-specific triggers)
  - Cookie session (Chromium maintains it across navigations)

Captures: every JSON response from /api/* endpoints on the same domain.
Saves: data/raw/domrf_full/<section>/<endpoint>.json (one JSON per endpoint).

Usage:
    PYTHONIOENCODING=utf-8 python data/sql/30_scrape_domrf.py [--headed] [--page <name>]

Run --headed first to verify WAF passes; switch to default headless after.
"""
import asyncio, json, os, re, sys
from urllib.parse import urlsplit, parse_qs, urlencode, urlunsplit
from playwright.async_api import async_playwright

ROOT = os.path.join(os.path.dirname(__file__), '..', 'raw', 'domrf_full')
BASE = 'https://xn--80az8a.xn--d1aqf.xn--p1ai'

# 12 analytics pages — name (used as folder) → URL path
PAGES = [
    ('housing',          '/аналитика/показатели_жилищного_строительства'),
    ('launch',           '/аналитика/запуски-и-вводы?repYear=2026&calculationType=SQUARE&period=YEAR'),
    ('share_construction','/аналитика/долевое_строительство'),
    ('housing_dev',      '/аналитика/жилищное_строительство'),
    ('escrow',           '/аналитика/эскроу'),
    ('realization',      '/аналитика/реализация_строящихся_квартир'),
    ('sold_ready',       '/аналитика/распроданность-стройготовность?repYear=2026&repMonth=3'),
    ('quartirografia',   '/аналитика/квартирография'),
    ('commissioning',    '/аналитика/ввод_жилья'),
    ('mortgage_rates',   '/аналитика/ставки_предложений_по_ипотеке'),
    ('mortgage_stats',   '/аналитика/ипотечное_кредитование'),
    ('stat_series',      '/аналитика/статистические_ряды'),
]


def safe_filename(url: str) -> str:
    """Turn API URL into safe filename: path + query → snake_case."""
    parts = urlsplit(url)
    base = parts.path.split('/api/')[-1].strip('/')
    base = re.sub(r'[^A-Za-z0-9_\-]', '_', base)[:100]
    if parts.query:
        q = re.sub(r'[^A-Za-z0-9_\-]', '_', parts.query)[:80]
        base = f'{base}__{q}'
    return base + '.json'


def upgrade_size(url: str) -> str:
    """If URL has size=15 (or any small value), upgrade to size=1000 to get full data."""
    parts = urlsplit(url)
    if not parts.query:
        return url
    qs = parse_qs(parts.query, keep_blank_values=True)
    if 'size' in qs:
        qs['size'] = ['1000']
    new_q = urlencode({k: v[0] for k, v in qs.items()})
    return urlunsplit((parts.scheme, parts.netloc, parts.path, new_q, parts.fragment))


# ---------------------------------------------------------------------------
# Page-specific lazy-load triggers
# ---------------------------------------------------------------------------
# Most DOM.RF analytics pages render Material-UI tabs/selects; data fetches
# fire on tab/option switch. The "kitchen sink" trigger walks every tab and
# every Select option, with networkidle waits in between. Pages with custom
# UI (tree pickers, multi-step filters) get dedicated triggers below.

async def _wait_quiet(page, ms=2500):
    try:
        await page.wait_for_load_state('networkidle', timeout=8000)
    except Exception:
        pass
    await page.wait_for_timeout(ms)


async def _click_all_tabs(page):
    """Click every Material-UI tab one-by-one, waiting between to capture XHR."""
    tab_count = await page.evaluate('''() => document.querySelectorAll(
        '[role="tab"], .MuiTab-root, button[data-tab]'
    ).length''')
    print(f'  tabs found: {tab_count}')
    for i in range(tab_count):
        try:
            await page.evaluate(
                '''(i) => {
                  const tabs = document.querySelectorAll(
                    '[role="tab"], .MuiTab-root, button[data-tab]'
                  );
                  if (tabs[i]) tabs[i].click();
                }''',
                i,
            )
            await _wait_quiet(page, 2000)
        except Exception as e:
            print(f'  tab[{i}] err: {e}')


async def _iterate_selects(page, max_options=20):
    """Open every MUI Select / native <select>, pick each option, wait."""
    select_count = await page.evaluate('''() => document.querySelectorAll(
        '[role="combobox"], [role="button"][aria-haspopup="listbox"], select'
    ).length''')
    print(f'  selects found: {select_count}')
    for i in range(select_count):
        try:
            opt_count = await page.evaluate(
                '''async (i) => {
                  const sels = document.querySelectorAll(
                    '[role="combobox"], [role="button"][aria-haspopup="listbox"], select'
                  );
                  const sel = sels[i];
                  if (!sel) return 0;
                  if (sel.tagName === 'SELECT') return sel.options.length;
                  sel.click();
                  await new Promise(r => setTimeout(r, 800));
                  return document.querySelectorAll('[role="option"], li[data-value]').length;
                }''',
                i,
            )
            opt_count = min(opt_count, max_options)
            for j in range(opt_count):
                try:
                    await page.evaluate(
                        '''async ([i, j]) => {
                          const sels = document.querySelectorAll(
                            '[role="combobox"], [role="button"][aria-haspopup="listbox"], select'
                          );
                          const sel = sels[i];
                          if (!sel) return;
                          if (sel.tagName === 'SELECT') {
                            sel.selectedIndex = j;
                            sel.dispatchEvent(new Event('change', {bubbles: true}));
                          } else {
                            sel.click();
                            await new Promise(r => setTimeout(r, 600));
                            const opts = document.querySelectorAll('[role="option"], li[data-value]');
                            if (opts[j]) opts[j].click();
                          }
                        }''',
                        [i, j],
                    )
                    await _wait_quiet(page, 1500)
                except Exception as e:
                    print(f'  select[{i}].opt[{j}] err: {e}')
                    break
        except Exception as e:
            print(f'  select[{i}] err: {e}')


async def _click_all_safe_buttons(page, deny=('Войти', 'Регистр', 'Выйти', 'Меню')):
    """Click every non-disabled button whose text doesn't match deny list."""
    deny_js = json.dumps(list(deny))
    btn_count = await page.evaluate(
        '''(deny) => Array.from(document.querySelectorAll('button:not([disabled])'))
            .filter(b => !deny.some(d => (b.textContent || '').includes(d))).length''',
        deny,
    )
    print(f'  safe buttons: {btn_count}')
    for i in range(min(btn_count, 30)):
        try:
            await page.evaluate(
                '''([i, deny]) => {
                  const bs = Array.from(document.querySelectorAll('button:not([disabled])'))
                    .filter(b => !deny.some(d => (b.textContent || '').includes(d)));
                  if (bs[i]) bs[i].click();
                }''',
                [i, list(deny)],
            )
            await _wait_quiet(page, 1500)
        except Exception:
            pass


async def trigger_kitchen_sink(page):
    """Generic: scroll, click all tabs, iterate all selects."""
    for _ in range(2):
        await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
        await page.wait_for_timeout(1200)
        await page.evaluate('window.scrollTo(0, 0)')
        await page.wait_for_timeout(800)
    await _click_all_tabs(page)
    await _iterate_selects(page, max_options=10)
    await _click_all_safe_buttons(page)


# ---------------------------------------------------------------------------
# Direct-API helpers — fetch from inside browser ctx so cookies/WAF survive
# ---------------------------------------------------------------------------
# These five pages don't fire XHR on user interaction — they're SSR-baked or
# expose a documented REST API. Instead of clicking we just call the APIs
# directly via page.evaluate(fetch()) and dump the JSON via the response hook.
# Page spec source: knowledge graph entities DomRF_Tab*_*_Apr26.

REGIONS_FULL_RU = list(range(1, 100))  # try region codes 1..99 (Rosstat-style)


async def _fetch_and_save(page, url, out_dir, seen_urls, saved):
    """Fetch URL inside browser context (so cookies/WAF stay valid), save body."""
    if url in seen_urls:
        return
    try:
        resp = await page.evaluate(
            '''async (u) => {
              const r = await fetch(u, {credentials: 'include'});
              const t = await r.text();
              return {status: r.status, ct: r.headers.get('content-type') || '', text: t};
            }''',
            url,
        )
    except Exception as e:
        print(f'  direct err {url}: {e}')
        return
    if not resp or resp.get('status', 0) >= 400:
        return
    text = resp.get('text') or ''
    if not text or len(text) < 20:
        return
    seen_urls.add(url)
    fname = safe_filename(url)
    with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
        f.write(text)
    saved.append((fname, len(text), url))
    print(f'  direct  {fname} ({len(text)}b)')


async def _dump_next_data(page, out_dir, saved):
    """Pull __NEXT_DATA__ JSON blob from the rendered HTML and save it."""
    blob = await page.evaluate('''() => {
      const el = document.getElementById('__NEXT_DATA__');
      return el ? el.textContent : null;
    }''')
    if not blob:
        return
    fname = '__NEXT_DATA__.json'
    with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
        f.write(blob)
    saved.append((fname, len(blob), 'inline:__NEXT_DATA__'))
    print(f'  ssr     {fname} ({len(blob)}b)')


async def trigger_housing(page, out_dir, seen_urls, saved):
    """Tab 1: показатели жилищного строительства — SSR + portal-analytics REST."""
    await _dump_next_data(page, out_dir, saved)
    base = '/portal-analytics/api/dashboard'
    # Aggregate (no filter) variants first
    for ep in ('by-region', 'by-developer', 'by-room-count', 'by-flat-area'):
        await _fetch_and_save(page, f'{base}/{ep}', out_dir, seen_urls, saved)
    # Per-region drilldown
    for rid in REGIONS_FULL_RU:
        await _fetch_and_save(page, f'{base}/by-region?regionId={rid}', out_dir, seen_urls, saved)
        await _fetch_and_save(page, f'{base}/by-room-count?regionId={rid}', out_dir, seen_urls, saved)
        await _fetch_and_save(page, f'{base}/by-flat-area?regionId={rid}', out_dir, seen_urls, saved)


async def trigger_housing_dev(page, out_dir, seen_urls, saved):
    """Tab 2: жилищное строительство — different Redux slice, same portal-analytics base."""
    await _dump_next_data(page, out_dir, saved)
    # Probe common analytics endpoints
    bases = [
        '/portal-analytics/api/housing',
        '/portal-analytics/api/dashboard',
        '/аналитика/api/housing',
    ]
    for b in bases:
        for ep in ('summary', 'by-region', 'by-developer', 'by-period', 'overview', 'main'):
            await _fetch_and_save(page, f'{b}/{ep}', out_dir, seen_urls, saved)
    await trigger_kitchen_sink(page)


async def trigger_realization(page, out_dir, seen_urls, saved):
    """Tab 6: реализация — /аналитика/api/rpp/* documented in graph."""
    await _dump_next_data(page, out_dir, saved)
    base = '/аналитика/api/rpp'
    rep_year, rep_month = 2026, 3
    # RF-level (no regionCode)
    for ep in ('total', 'housing', 'readyYear'):
        await _fetch_and_save(
            page, f'{base}/{ep}?repMonth={rep_month}&repYear={rep_year}',
            out_dir, seen_urls, saved,
        )
    await _fetch_and_save(
        page,
        f'{base}/developer?typeSquare=total&repMonth={rep_month}&repYear={rep_year}'
        f'&developerOrder=totalSquare:desc',
        out_dir, seen_urls, saved,
    )
    # Per-region (89 регионов РФ — codes mostly in 1..99)
    for rcode in REGIONS_FULL_RU:
        for ep in ('total', 'housing', 'readyYear'):
            await _fetch_and_save(
                page,
                f'{base}/{ep}?regionCode={rcode}&repMonth={rep_month}&repYear={rep_year}',
                out_dir, seen_urls, saved,
            )
        for ts in ('total', 'living'):
            await _fetch_and_save(
                page,
                f'{base}/developer?typeSquare={ts}&regionCode={rcode}'
                f'&repMonth={rep_month}&repYear={rep_year}'
                f'&developerOrder=totalSquare:desc',
                out_dir, seen_urls, saved,
            )


async def trigger_mortgage_rates(page, out_dir, seen_urls, saved):
    """Tab 10: ставки предложений по ипотеке — likely SSR; probe known APIs."""
    await _dump_next_data(page, out_dir, saved)
    bases = [
        '/portal-analytics/api/mortgage-rates',
        '/portal-analytics/api/mortgage',
        '/аналитика/api/mortgage-rates',
        '/аналитика/api/mortgage',
    ]
    for b in bases:
        for ep in ('', '/list', '/banks', '/offers', '/dashboard', '/summary'):
            await _fetch_and_save(page, f'{b}{ep}', out_dir, seen_urls, saved)
    await trigger_kitchen_sink(page)


async def trigger_stat_series(page, out_dir, seen_urls, saved):
    """Tab 12: статистические ряды — index of XLSX files baked into __NEXT_DATA__.

    Per graph: data lives in props.initialState.housingData.list (CMS hierarchy).
    XLSX URLs: /site/binaries/content/assets/domrf/xlsdashboard/{filename}.xlsx
    Strategy: dump __NEXT_DATA__ + extract every .xlsx href and download.
    """
    await _dump_next_data(page, out_dir, saved)
    # Extract all .xlsx links from the rendered DOM and the __NEXT_DATA__ blob
    links = await page.evaluate('''() => {
      const set = new Set();
      document.querySelectorAll('a[href]').forEach(a => {
        const h = a.getAttribute('href');
        if (h && h.toLowerCase().includes('.xlsx')) set.add(h);
      });
      const blob = document.getElementById('__NEXT_DATA__');
      if (blob) {
        const re = /["'](\\/site\\/binaries[^"']+\\.xlsx)["']/g;
        let m;
        while ((m = re.exec(blob.textContent)) !== null) set.add(m[1]);
      }
      return Array.from(set);
    }''')
    print(f'  xlsx links found: {len(links)}')
    xlsx_dir = os.path.join(out_dir, 'xlsx')
    os.makedirs(xlsx_dir, exist_ok=True)
    for href in links:
        url = href if href.startswith('http') else BASE + href
        # Download via fetch in browser context, then write bytes locally
        try:
            b64 = await page.evaluate(
                '''async (u) => {
                  const r = await fetch(u, {credentials: 'include'});
                  if (!r.ok) return null;
                  const buf = await r.arrayBuffer();
                  let s = '';
                  const arr = new Uint8Array(buf);
                  for (let i = 0; i < arr.length; i++) s += String.fromCharCode(arr[i]);
                  return btoa(s);
                }''',
                url,
            )
            if not b64:
                continue
            import base64
            data = base64.b64decode(b64)
            fname = re.sub(r'[^A-Za-z0-9_.\-]', '_', os.path.basename(href.split('?')[0]))
            with open(os.path.join(xlsx_dir, fname), 'wb') as f:
                f.write(data)
            saved.append((f'xlsx/{fname}', len(data), url))
            print(f'  xlsx    {fname} ({len(data)}b)')
        except Exception as e:
            print(f'  xlsx err {href}: {e}')


PAGE_TRIGGERS = {
    'housing':        trigger_housing,
    'housing_dev':    trigger_housing_dev,
    'realization':    trigger_realization,
    'mortgage_rates': trigger_mortgage_rates,
    'stat_series':    trigger_stat_series,
}


async def scrape_page(browser, name, path):
    out_dir = os.path.join(ROOT, name)
    os.makedirs(out_dir, exist_ok=True)
    print(f'\n=== {name} ({path}) ===')
    context = await browser.new_context(
        user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
                   '(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36',
        locale='ru-RU',
        viewport={'width': 1920, 'height': 1080},
    )
    page = await context.new_page()

    seen_urls = set()
    saved = []

    async def on_response(resp):
        url = resp.url
        # Match same-origin /api/ endpoints, skip yandex/sendsay trackers
        if 'xn--80az8a.xn--d1aqf.xn--p1ai' not in url or '/api/' not in url:
            return
        ct = (resp.headers.get('content-type') or '').lower()
        if 'json' not in ct:
            return
        if url in seen_urls:
            return
        seen_urls.add(url)
        try:
            text = await resp.text()
            fname = safe_filename(url)
            with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
                f.write(text)
            saved.append((fname, len(text), url))
            print(f'  saved {fname} ({len(text)}b)')
        except Exception as e:
            print(f'  ERR {url}: {e}')

    page.on('response', lambda r: asyncio.create_task(on_response(r)))

    full_url = BASE + path
    try:
        await page.goto(full_url, wait_until='domcontentloaded', timeout=60_000)
        try:
            await page.wait_for_load_state('networkidle', timeout=30_000)
        except Exception:
            pass
        await page.wait_for_timeout(5000)

        # Per-page triggers (falls back to kitchen sink for everything new)
        trigger = PAGE_TRIGGERS.get(name)
        try:
            if trigger is None:
                await trigger_kitchen_sink(page)
            else:
                # New triggers take (page, out_dir, seen_urls, saved) so they
                # can perform direct API fetches and dump SSR blobs themselves.
                await trigger(page, out_dir, seen_urls, saved)
        except Exception as e:
            print(f'  trigger err: {e}')

        # Final settle
        await _wait_quiet(page, 3000)

        # Re-fetch with size=1000 for endpoints that have small page-size
        for url in list(seen_urls):
            up = upgrade_size(url)
            if up != url and up not in seen_urls:
                try:
                    resp = await page.evaluate(
                        '''(u) => fetch(u, {credentials:'include'})
                            .then(r => r.text()).then(t => ({status: 200, text: t}))''',
                        up,
                    )
                    if resp and resp.get('text'):
                        seen_urls.add(up)
                        fname = safe_filename(up)
                        with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
                            f.write(resp['text'])
                        saved.append((fname, len(resp['text']), up))
                        print(f'  upgraded {fname} ({len(resp["text"])}b)')
                except Exception as e:
                    print(f'  upgrade err {up}: {e}')
    finally:
        await context.close()
    return saved


async def main():
    headed = '--headed' in sys.argv
    target = None
    if '--page' in sys.argv:
        target = sys.argv[sys.argv.index('--page') + 1]

    pages = [(n, p) for n, p in PAGES if target is None or n == target]
    print(f'Scraping {len(pages)} pages, headed={headed}')
    os.makedirs(ROOT, exist_ok=True)

    async with async_playwright() as pw:
        browser = await pw.chromium.launch(headless=not headed)
        all_saved = {}
        for name, path in pages:
            try:
                all_saved[name] = await scrape_page(browser, name, path)
            except Exception as e:
                print(f'PAGE {name} FAILED: {e}')
                all_saved[name] = []
        await browser.close()

    print('\n=== SUMMARY ===')
    for name, saved in all_saved.items():
        total = sum(s[1] for s in saved)
        print(f'  {name:22s} files={len(saved):3d} total={total:>10}b')


if __name__ == '__main__':
    asyncio.run(main())