"""Cian Valuation backfill через real Chrome (Playwright) с auth cookies.

Iterates listings без свежей external_valuations записи, открывает Calculator URL
с реальным address+area+rooms, ждёт когда state.estimation.sale.data populated,
extract'ит sale/rent/chart/houseId, INSERT'ит в external_valuations.

Requires:
  pip install playwright "psycopg[binary]"
  python -m playwright install chromium
  SSH tunnel to tradein-postgres on localhost:15433
  Env: TRADEIN_DB_DSN, COOKIE_ENCRYPTION_KEY

Run:
  python C:\\Users\\user\\cian-backfill\\playwright_valuation.py --batch 10 --delay 3 --wait 8
"""
from __future__ import annotations

import argparse
import asyncio
import hashlib
import json
import os
import re
import signal
import sys
import time
import urllib.parse
from datetime import datetime, timezone

import psycopg
from playwright.async_api import async_playwright

# JS — wait + extract когда valuation populated (или timeout)
EXTRACT_JS = """
() => {
  const arr = window._cianConfig && window._cianConfig['valuation-for-agent-frontend'];
  if (!arr) return { error: 'no MFE', captcha: location.href.includes('captcha') };
  const entry = arr.find(e => e && e.key === 'initialState');
  if (!entry || !entry.value) return { error: 'no initialState' };
  const v = entry.value;
  // unwrap state.data контейнеры в чистые объекты (Cian шаблон — { data, isFetching, isError })
  const unwrap = (o) => (o && typeof o === 'object' && 'data' in o ? o.data : o);
  return {
    isAuth: v.user && v.user.isAuthenticated,
    user_id_cian: v.user && v.user.userId,
    isFetching: (v.estimation && v.estimation.sale && v.estimation.sale.isFetching) ||
                (v.estimationChart && v.estimationChart.isFetching),
    sale: v.estimation && v.estimation.sale && v.estimation.sale.data,
    rent: v.estimation && v.estimation.rent && v.estimation.rent.data,
    chart_obj: v.estimationChart && v.estimationChart.data,
    house: unwrap(v.houseInfo),
    management_company: unwrap(v.managementCompany),
    nearby_houses_info: unwrap(v.nearbyHousesInfo),
    market_trend: unwrap(v.marketTrand),  // sic — typo в Cian API
    offers_history: unwrap(v.offersHistory),
    precise_filters: v.preciseFilters,
    geocode: unwrap(v.geocode),
    services: v.services,
    house_id: v.filters && v.filters.houseId,
    filters_address: v.filters && v.filters.address,
    filters_house_id: v.filters && v.filters.houseId,
    // FULL initialState — для raw_payload (полный snapshot для будущего парсинга)
    full_state: v,
  };
}
"""

WAIT_JS = """
() => {
  const arr = window._cianConfig && window._cianConfig['valuation-for-agent-frontend'];
  if (!arr) return false;
  const entry = arr.find(e => e && e.key === 'initialState');
  if (!entry || !entry.value) return false;
  const v = entry.value;
  const sale = v.estimation && v.estimation.sale;
  // Готово когда либо есть data, либо isError, и не isFetching
  if (sale && sale.data && sale.data.price) return true;
  if (sale && sale.isError) return true;
  return false;
}
"""

# Cian floor enum
_FLOOR_SPECIAL = {1: "floorOne", 2: "floorTwo"}


def cian_floor_enum(floor: int, total_floors: int | None) -> str:
    if floor in _FLOOR_SPECIAL:
        return _FLOOR_SPECIAL[floor]
    if total_floors and floor == total_floors:
        return "floorLast"
    return "floorOther"


def compute_cache_key(address, total_area, rooms, floor, repair, deal):
    norm = f"{address.strip().lower()}|{total_area}|{rooms}|{floor}|{repair}|{deal}"
    return hashlib.sha256(norm.encode()).hexdigest()


def parse_change_pct(value_str: str) -> float | None:
    """'6,4%' / '-1.2%' → 6.4 / -1.2"""
    if not value_str:
        return None
    m = re.search(r"-?\d+[\.,]?\d*", value_str)
    if not m:
        return None
    return float(m.group(0).replace(",", "."))


def normalize_address(addr: str) -> str:
    """listings.address типа 'улица Машинная, 1В/4 · р-н Ленинский' → 'Екатеринбург, улица Машинная, 1В/4'.

    1. отрезаем ' · р-н ...' postfix
    2. подставляем 'Екатеринбург, ' если города нет в строке
    """
    if not addr:
        return addr
    # cut at middle-dot (Cian district separator)
    addr = re.split(r"\s*[·•]\s*", addr)[0].strip().rstrip(",")
    if "екатеринбург" not in addr.lower():
        addr = f"Екатеринбург, {addr}"
    return addr


def build_calc_url(address: str, area: float, rooms: int, floor: int, total_floors: int | None) -> str:
    params = {
        "address": normalize_address(address),
        "totalArea": str(area),
        "roomsCount": str(rooms),
        "valuationType": "sale",
        "floor[0]": cian_floor_enum(floor, total_floors),
        "repairType[0]": "repairTypeCosmetic",
    }
    return f"https://www.cian.ru/kalkulator-nedvizhimosti/?{urllib.parse.urlencode(params, safe='[]')}"


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--batch", type=int, default=10)
    ap.add_argument("--delay", type=float, default=3.0, help="seconds между listings")
    ap.add_argument("--wait", type=float, default=8.0, help="max seconds wait until valuation data")
    ap.add_argument("--max", type=int, default=5500)
    ap.add_argument("--headed", action="store_true")
    args = ap.parse_args()

    dsn = os.environ.get("TRADEIN_DB_DSN")
    if not dsn:
        sys.exit("FATAL: TRADEIN_DB_DSN env required")
    enc_key = os.environ.get("COOKIE_ENCRYPTION_KEY")
    if not enc_key:
        sys.exit("FATAL: COOKIE_ENCRYPTION_KEY env required")
    pg_dsn = dsn.replace("postgresql+psycopg://", "postgresql://")

    interrupted = False

    def on_sigint(s, f):
        nonlocal interrupted
        print("\n!! Ctrl+C — finishing current item")
        interrupted = True
    signal.signal(signal.SIGINT, on_sigint)

    async def run() -> None:
        nonlocal interrupted
        cum_ok = cum_fail = cum_skipped = 0
        start = time.time()
        last_id = 0

        def load_cookies() -> tuple[list, datetime | None]:
            with psycopg.connect(pg_dsn) as conn:
                with conn.cursor() as cur:
                    cur.execute(
                        """
                        SELECT pgp_sym_decrypt(cookies_encrypted, %s)::text, uploaded_at
                        FROM cian_session_cookies
                        WHERE expires_at_estimate > NOW() AND last_invalid_at IS NULL
                        ORDER BY uploaded_at DESC LIMIT 1
                        """,
                        (enc_key,),
                    )
                    r = cur.fetchone()
                    if not r:
                        return [], None
                    jar = json.loads(r[0])
                    out = []
                    for name, val in jar.items():
                        out.append({
                            "name": name, "value": val,
                            "domain": ".cian.ru", "path": "/",
                            "secure": True, "sameSite": "Lax",
                        })
                    return out, r[1]

        cookies_from_db, cookies_uploaded_at = load_cookies()
        if not cookies_from_db:
            sys.exit("FATAL: no valid cookies in cian_session_cookies")
        print(f"[init] loaded {len(cookies_from_db)} cookies (uploaded {cookies_uploaded_at})")

        profile_dir = os.environ.get("CIAN_PROFILE_DIR", r"C:\Users\user\cian-backfill\chrome_profile")
        os.makedirs(profile_dir, exist_ok=True)

        async with async_playwright() as p:
            ctx = await p.chromium.launch_persistent_context(
                user_data_dir=profile_dir,
                headless=not args.headed,
                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
                viewport={"width": 1366, "height": 900},
                locale="ru-RU",
                args=["--disable-blink-features=AutomationControlled"],
            )
            # First launch: inject cookies из БД. Subsequent launches: cookies уже в profile.
            existing = await ctx.cookies("https://www.cian.ru")
            has_auth = any(c.get("name") == "DMIR_AUTH" for c in existing)
            if not has_auth and cookies_from_db:
                await ctx.add_cookies(cookies_from_db)
                print(f"[init] injected {len(cookies_from_db)} cookies into fresh profile")
            else:
                print(f"[init] reusing profile cookies ({len(existing)} present, DMIR_AUTH={'yes' if has_auth else 'no'})")
            page = ctx.pages[0] if ctx.pages else await ctx.new_page()
            page.set_default_navigation_timeout(45_000)
            page.set_default_timeout(45_000)

            processed = 0
            while not interrupted and processed < args.max:
                # 1. Fetch batch of listings WITH полный address + geo
                with psycopg.connect(pg_dsn) as conn:
                    with conn.cursor() as cur:
                        cur.execute(
                            """
                            SELECT l.id, l.address, l.area_m2, l.rooms, l.floor, l.total_floors
                            FROM listings l
                            WHERE l.source = 'cian'
                              AND l.address IS NOT NULL
                              AND length(l.address) > 15  -- хотя бы город+улица+номер
                              AND l.area_m2 IS NOT NULL
                              AND l.rooms IS NOT NULL
                              AND l.floor IS NOT NULL
                              AND l.total_floors IS NOT NULL
                              AND l.id > %s
                              AND NOT EXISTS (
                                SELECT 1 FROM external_valuations ev
                                WHERE ev.source='cian_valuation'
                                  AND ev.listing_id = l.id
                                  AND ev.expires_at > NOW()
                              )
                            ORDER BY l.id
                            LIMIT %s
                            """,
                            (last_id, args.batch),
                        )
                        rows = cur.fetchall()
                if not rows:
                    print(f"\n[done] drained — no more listings")
                    break
                print(f"\n[batch] {len(rows)} listings (cum: ok={cum_ok} fail={cum_fail} skipped={cum_skipped})")

                with psycopg.connect(pg_dsn, autocommit=False) as conn:
                    for (lid, addr, area, rooms, floor, total_floors) in rows:
                        if interrupted:
                            break
                        processed += 1
                        last_id = lid
                        t0 = time.time()
                        try:
                            area_f = float(area)
                            rooms_i = int(rooms)
                            floor_i = int(floor)
                            total_floors_i = int(total_floors) if total_floors else None
                            url = build_calc_url(addr, area_f, rooms_i, floor_i, total_floors_i)
                            await page.goto(url, wait_until="domcontentloaded")
                            # Click submit button "Оценить" — без этого Cian не вычисляет
                            try:
                                btn = page.get_by_test_id("next-button")
                                await btn.click(timeout=5000)
                            except Exception:
                                pass  # button может отсутствовать если URL params не валидны — fallback
                            try:
                                await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000))
                            except Exception:
                                pass  # fall through и попробуем extract — может data есть с задержкой
                            data = await page.evaluate(EXTRACT_JS)
                            if data.get("error"):
                                if data.get("captcha"):
                                    print(f"  [{lid}] CAPTCHA — STOP, please solve in --headed mode")
                                    interrupted = True
                                    break
                                cum_fail += 1
                                print(f"  [{lid}] err={data['error']} {time.time() - t0:.1f}s")
                                continue
                            if not data.get("isAuth"):
                                # 1. Try fresher cookies из БД
                                fresh, uploaded = load_cookies()
                                if fresh and uploaded and (cookies_uploaded_at is None or uploaded > cookies_uploaded_at):
                                    print(f"  [{lid}] auth lost — reloading fresh cookies (uploaded {uploaded})")
                                    await ctx.clear_cookies()
                                    await ctx.add_cookies(fresh)
                                    cookies_uploaded_at = uploaded
                                    await page.goto(url, wait_until="domcontentloaded")
                                    try:
                                        await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000))
                                    except Exception:
                                        pass
                                    data = await page.evaluate(EXTRACT_JS)
                                # 2. Если headed — даём user'у залогиниться вручную в открытом окне
                                if not data.get("isAuth") and args.headed:
                                    print(f"\n  [{lid}] !! AUTH LOST !!")
                                    print(f"  → В открытом Chrome окне открой www.cian.ru, ЗАЛОГИНЬСЯ.")
                                    print(f"  → После успешного логина нажми Enter здесь чтобы продолжить.")
                                    print(f"  → Или Ctrl+C чтобы остановить.")
                                    try:
                                        # blocking input — пользователь логинится сам
                                        loop = asyncio.get_event_loop()
                                        await loop.run_in_executor(None, input, "  >>> press Enter when logged in: ")
                                    except (EOFError, KeyboardInterrupt):
                                        interrupted = True
                                        break
                                    # retry same listing
                                    await page.goto(url, wait_until="domcontentloaded")
                                    try:
                                        await page.wait_for_function(WAIT_JS, timeout=int(args.wait * 1000))
                                    except Exception:
                                        pass
                                    data = await page.evaluate(EXTRACT_JS)
                                # 3. Headless и всё ещё no auth → STOP
                                if not data.get("isAuth"):
                                    print(f"  [{lid}] not auth — STOP. Запусти с --headed и login manually.")
                                    interrupted = True
                                    break
                            sale = data.get("sale")
                            if not sale or not sale.get("price"):
                                cum_skipped += 1
                                print(f"  [{lid}] no sale data (address not matched?) {time.time() - t0:.1f}s")
                                continue
                            rent = data.get("rent") or {}
                            chart_obj = data.get("chart_obj") or {}
                            chart_points = (chart_obj.get("chartData") or {}).get("data") or []
                            chart_title = chart_obj.get("title") or {}
                            chart_change_dir = chart_title.get("change")  # 'increase'/'decrease'/'neutral'
                            chart_change_pct = parse_change_pct(chart_title.get("changeValue") or "")
                            house_id = data.get("house_id")
                            cache_key = compute_cache_key(addr, area_f, rooms_i, floor_i, "cosmetic", "sale")

                            with conn.cursor() as cur:
                                cur.execute(
                                    """
                                    INSERT INTO external_valuations (
                                        source, cache_key, address,
                                        total_area, rooms_count, floor, total_floors,
                                        repair_type, deal_type,
                                        price_rub, accuracy,
                                        low_price, high_price,
                                        house_id, listing_id,
                                        sale_price_rub, sale_accuracy,
                                        rent_price_rub, rent_accuracy,
                                        chart, chart_change_pct, chart_change_direction,
                                        external_house_id, filters_hash,
                                        raw_payload, fetched_at, expires_at
                                    ) VALUES (
                                        'cian_valuation', %s, %s,
                                        CAST(%s AS numeric), %s, %s, %s,
                                        'cosmetic', 'sale',
                                        CAST(%s AS numeric), CAST(%s AS numeric),
                                        CAST(%s AS numeric), CAST(%s AS numeric),
                                        NULL, CAST(%s AS bigint),
                                        CAST(%s AS numeric), CAST(%s AS numeric),
                                        CAST(%s AS numeric), CAST(%s AS numeric),
                                        CAST(%s AS jsonb), CAST(%s AS numeric), %s,
                                        %s, NULL,
                                        CAST(%s AS jsonb), NOW(), NOW() + INTERVAL '30 days'
                                    )
                                    ON CONFLICT (source, cache_key) DO UPDATE SET
                                        price_rub             = EXCLUDED.price_rub,
                                        accuracy              = EXCLUDED.accuracy,
                                        low_price             = EXCLUDED.low_price,
                                        high_price            = EXCLUDED.high_price,
                                        listing_id            = COALESCE(EXCLUDED.listing_id, external_valuations.listing_id),
                                        sale_price_rub        = EXCLUDED.sale_price_rub,
                                        sale_accuracy         = EXCLUDED.sale_accuracy,
                                        rent_price_rub        = EXCLUDED.rent_price_rub,
                                        rent_accuracy         = EXCLUDED.rent_accuracy,
                                        chart                 = EXCLUDED.chart,
                                        chart_change_pct      = EXCLUDED.chart_change_pct,
                                        chart_change_direction = EXCLUDED.chart_change_direction,
                                        external_house_id     = EXCLUDED.external_house_id,
                                        raw_payload           = EXCLUDED.raw_payload,
                                        fetched_at            = NOW(),
                                        expires_at            = NOW() + INTERVAL '30 days'
                                    """,
                                    (
                                        cache_key, addr,
                                        area_f, rooms_i, floor_i, total_floors_i,
                                        sale.get("price"), (sale.get("accuracy") or 0) / 100,
                                        sale.get("priceFrom"), sale.get("priceTo"),
                                        lid,
                                        sale.get("price"), (sale.get("accuracy") or 0) / 100,
                                        rent.get("price"), (rent.get("accuracy") or 0) / 100 if rent.get("accuracy") else None,
                                        json.dumps(chart_points), chart_change_pct,
                                        chart_change_dir if chart_change_dir in ("increase", "decrease", "neutral") else None,
                                        house_id,
                                        json.dumps(data.get("full_state") or {"sale": sale, "rent": rent, "chart": chart_obj, "house_id": house_id}),
                                    ),
                                )
                                conn.commit()
                            cum_ok += 1
                            print(f"  [{lid}] OK sale={sale.get('price'):,} rent={rent.get('price')} chart={len(chart_points)} chg={chart_change_pct}%/{chart_change_dir} houseId={house_id} {time.time() - t0:.1f}s")
                        except Exception as exc:
                            conn.rollback()
                            cum_fail += 1
                            print(f"  [{lid}] ERR {type(exc).__name__}: {str(exc)[:140]} {time.time() - t0:.1f}s")
                        await asyncio.sleep(args.delay)

            await ctx.close()

        print()
        print(f"== Summary ==")
        print(f"   ok={cum_ok} fail={cum_fail} skipped_no_data={cum_skipped}")
        print(f"   elapsed={time.time() - start:.1f}s  finished={datetime.now(timezone.utc).isoformat(timespec='seconds')}")

    asyncio.run(run())
    return 0


if __name__ == "__main__":
    sys.exit(main())
