"""Playwright-based Avito sweep — обходит Qrator JS challenge через real headless Chromium.

Use case: curl_cffi заблочен Qrator anti-DDoS (429 + captcha challenge). Real
browser proходит JS challenge автоматически.

Setup:
    cd tradein-mvp/backend
    uv pip install playwright
    uv run playwright install chromium

Usage:
    export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
    uv run --project tradein-mvp/backend python tradein-mvp/scripts/avito-playwright-sweep.py
    # options: --pages N --delay S --mode {citywide|byrooms} --headless
"""
from __future__ import annotations

# Banner FIRST — даже до imports чтобы видно что скрипт стартанул:
import sys as _sys

print("=" * 60, flush=True)
print("AVITO PLAYWRIGHT SWEEP — starting...", flush=True)
print("=" * 60, flush=True)
_sys.stdout.flush()

import argparse
import asyncio
import json
import os
import random
import sys
import time
import types
from pathlib import Path
from urllib.parse import urlencode

# Force unbuffered output
os.environ["PYTHONUNBUFFERED"] = "1"

_BACKEND = Path(__file__).resolve().parent.parent / "backend"
if str(_BACKEND) not in sys.path:
    sys.path.insert(0, str(_BACKEND))

# WeasyPrint stub (Windows без GTK)
for _m in ("weasyprint", "weasyprint.text", "weasyprint.text.ffi", "weasyprint.css", "weasyprint.css.targets"):
    sys.modules[_m] = types.ModuleType(_m)
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})

from playwright.async_api import Browser, BrowserContext, async_playwright  # noqa: E402

try:
    from playwright_stealth import Stealth  # noqa: E402
    _STEALTH_AVAILABLE = True
except ImportError:
    _STEALTH_AVAILABLE = False
    print("[warn] playwright-stealth not installed — без stealth, может triggered больше CAPTCHA", flush=True)

from app.core.db import SessionLocal  # noqa: E402
from app.services.scrapers.avito import AvitoScraper  # noqa: E402
from app.services.scrapers.base import save_listings  # noqa: E402

COOKIES_FILE = Path(__file__).resolve().parent / ".avito-cookies.json"
AVITO_BASE = "https://www.avito.ru"
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/132.0.0.0 Safari/537.36"

# Reuse AvitoScraper's parser (DOM cards extraction)
_parser_instance = AvitoScraper()
_parser_instance._anchor_lat = 0.0  # noqa: SLF001
_parser_instance._anchor_lon = 0.0  # noqa: SLF001
_parser_instance._anchor_radius_m = 1000  # noqa: SLF001


def load_cookies() -> list[dict]:
    """Load DevTools-exported cookies → Playwright format."""
    if not COOKIES_FILE.exists():
        print(f"WARN: cookies file not found at {COOKIES_FILE} — будем без cookies")
        return []
    raw = json.loads(COOKIES_FILE.read_text(encoding="utf-8"))
    out = []
    for c in raw:
        ck = {
            "name": c["name"],
            "value": c["value"],
            "domain": c["domain"],
            "path": c.get("path", "/"),
            "secure": c.get("secure", True),
            "httpOnly": c.get("httpOnly", False),
        }
        if c.get("expirationDate"):
            ck["expires"] = int(c["expirationDate"])
        ss = c.get("sameSite")
        if ss == "no_restriction":
            ck["sameSite"] = "None"
        elif ss == "lax":
            ck["sameSite"] = "Lax"
        elif ss == "strict":
            ck["sameSite"] = "Strict"
        out.append(ck)
    return out


def build_citywide_url(page: int) -> str:
    return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?{urlencode({'s':104,'p':page})}"


ROOM_SLUGS: list[tuple[str, str]] = [
    ("студии",       "studii-ASgBAgICAkSSA8YQygj~WA"),
    ("1-комн.",      "1-komnatnye-ASgBAgICAkSSA8YQygiAWQ"),
    ("2-комн.",      "2-komnatnye-ASgBAgICAkSSA8YQygiCWQ"),
    ("3-комн.",      "3-komnatnye-ASgBAgICAkSSA8YQygiEWQ"),
    ("4-комн.",      "4-komnatnye-ASgBAgICAkSSA8YQygiGWQ"),
    ("5+комн.",      "5-komnatnye-ASgBAgICAkSSA8YQygiIWQ"),
    ("своб.планир.", "svobodnaya_planirovka-ASgBAgICAkSSA8YQygj8zzI"),
]


def build_rooms_url(slug: str, page: int) -> str:
    return f"{AVITO_BASE}/ekaterinburg/kvartiry/prodam/{slug}?{urlencode({'s':104,'p':page})}"


async def fetch_page(context: BrowserContext, url: str) -> tuple[int, str | None]:
    """Open page in Playwright, wait for content, return (status, html or None).

    Strategy:
    1. goto с wait_until='domcontentloaded' (быстро, не ждём всех scripts)
    2. Если status 200 → wait_for_selector data-marker=item (Qrator challenge должен resolve пока)
    3. Иначе если 429/403 — challenge не прошёл, дать ещё 5s + retry content (maybe challenge resolves async)
    """
    page = await context.new_page()
    try:
        try:
            response = await page.goto(url, wait_until="domcontentloaded", timeout=45000)
        except Exception as e:  # noqa: BLE001
            print(f"  goto FAIL: {type(e).__name__}: {str(e)[:80]}")
            return 0, None
        status = response.status if response else 0
        if status in (429, 403):
            # Qrator challenge — wait для JS auto-resolve, потом reload
            await asyncio.sleep(5)
            try:
                response = await page.reload(wait_until="domcontentloaded", timeout=30000)
                status = response.status if response else status
            except Exception:  # noqa: BLE001
                pass
        if status != 200:
            return status, None
        # Wait for item cards
        try:
            await page.wait_for_selector('[data-marker="item"]', timeout=15000)
        except Exception:  # noqa: BLE001
            pass
        html = await page.content()
        return status, html
    finally:
        await page.close()


async def sweep_urls(
    urls_with_label: list[tuple[str, str]],
    delay: float,
    headless: bool,
) -> int:
    """Generic sweep: list of (label, url) → fetch + parse + save per URL."""
    cookies = load_cookies()
    print(f"loaded {len(cookies)} cookies")
    total_fetched = 0
    total_ins = 0
    total_upd = 0
    challenge_hits = 0
    print("=== Launching Playwright browser ===", flush=True)
    async with async_playwright() as p:
        print(f"  headless={headless}", flush=True)
        browser: Browser = await p.chromium.launch(
            headless=headless,
            args=[
                "--disable-blink-features=AutomationControlled",
                "--no-sandbox",
            ],
        )
        print("  browser launched, creating context...", flush=True)
        context: BrowserContext = await browser.new_context(
            user_agent=UA,
            locale="ru-RU",
            timezone_id="Asia/Yekaterinburg",
            viewport={"width": 1366, "height": 900},
        )
        # Stealth — optional, skip on error
        if _STEALTH_AVAILABLE:
            try:
                stealth = Stealth()
                await stealth.apply_stealth_async(context)
                print("  stealth applied", flush=True)
            except Exception as e:  # noqa: BLE001
                print(f"  stealth skipped (err): {type(e).__name__}: {e}", flush=True)
        if cookies:
            await context.add_cookies(cookies)
            print(f"  cookies added ({len(cookies)})", flush=True)

        # Initial warmup: open main Avito + wait для user pass CAPTCHA если нужно
        print("\n=== WARMUP: opening Avito initial page ===", flush=True)
        print("  creating new page tab...", flush=True)
        warmup_page = await context.new_page()
        print("  navigating to Avito (timeout 30s)...", flush=True)
        try:
            await warmup_page.goto(
                "https://www.avito.ru/ekaterinburg/kvartiry/prodam-ASgBAgICAUSSA8YQ?s=104&p=1",
                wait_until="domcontentloaded",
                timeout=30000,
            )
            print("  goto done, reading content...", flush=True)
        except Exception as e:  # noqa: BLE001
            print(f"  goto FAIL: {type(e).__name__}: {e}", flush=True)
        # Check if challenge page — reliable detector: NO listing cards в HTML.
        # Слово "captcha" может быть в JS embed страницы normally (false positive).
        warmup_html = await warmup_page.content()
        print(f"  content read: {len(warmup_html)} bytes", flush=True)
        try:
            await warmup_page.wait_for_selector('[data-marker="item"]', timeout=5000)
            has_items = True
        except Exception:  # noqa: BLE001
            has_items = False
        is_qrator = "qrator" in warmup_html.lower() and len(warmup_html) < 100000
        challenge_present = (not has_items) and (is_qrator or len(warmup_html) < 50000)
        print(f"  has_items={has_items} qrator={is_qrator} size={len(warmup_html)} challenge={challenge_present}", flush=True)
        if challenge_present:
            print("\n🚨 CAPTCHA detected на initial page!")
            print("👉 Окно браузера открыто — пройди капчу руками,")
            print("👉 затем нажми Enter здесь чтобы продолжить sweep...")
            try:
                input(">>> Press ENTER when ready: ")
            except (EOFError, KeyboardInterrupt):
                print("aborted")
                await browser.close()
                return 0
            # Verify challenge passed
            await warmup_page.reload(wait_until="domcontentloaded", timeout=30000)
            new_html = await warmup_page.content()
            if any(m in new_html.lower() for m in ("captcha", "qrator")):
                print("⚠️ всё ещё CAPTCHA — повторно пройди или Ctrl+C")
                input(">>> Retry ENTER: ")
            else:
                print("✅ CAPTCHA пройдена, продолжаю sweep")
        else:
            print("✅ Initial page OK (no challenge)")
        await warmup_page.close()

        try:
            # Per-category skip flag: 0 lots на странице категории = end of pagination
            # ДЛЯ ТЕКУЩЕЙ КАТЕГОРИИ, а не для всего sweep'а (был bug: break ломал всё).
            current_cat: str | None = None
            skip_until_next_cat = False
            for i, (label, url) in enumerate(urls_with_label, 1):
                # Категория = всё до " page=" в label ("1-комн. page=5/60" → "1-комн.")
                cat = label.split(" page=")[0] if " page=" in label else label
                if cat != current_cat:
                    current_cat = cat
                    skip_until_next_cat = False
                    challenge_hits = 0  # reset 429-counter на границе категорий
                if skip_until_next_cat:
                    continue

                print(f"[{i}/{len(urls_with_label)}] {label}", flush=True)
                status, html = await fetch_page(context, url)
                if status != 200 or not html:
                    print(f"  HTTP {status} → skip", flush=True)
                    if status in (429, 403):
                        challenge_hits += 1
                        if challenge_hits >= 3:
                            print(f"  ⚠️ {challenge_hits}x challenge hits — sleep 60s extra", flush=True)
                            print("  если CAPTCHA в окне — пройди и нажми ENTER:", flush=True)
                            try:
                                input("  >>> ")
                            except (EOFError, KeyboardInterrupt):
                                break
                            challenge_hits = 0
                    await asyncio.sleep(delay)
                    continue
                # Parse + save
                lots = _parser_instance._parse_html(html, source_url_base=url)  # noqa: SLF001
                if not lots:
                    print(f"  0 lots — end of pagination for {cat!r}", flush=True)
                    if " page=" in label:
                        # 0 lots на пагинированной странице = end of category, не sweep
                        skip_until_next_cat = True
                    continue
                db = SessionLocal()
                try:
                    ins, upd = save_listings(db, lots)
                    total_fetched += len(lots)
                    total_ins += ins
                    total_upd += upd
                    print(
                        f"  {len(lots)} lots → ins={ins} upd={upd} "
                        f"[cum: tot={total_fetched} new={total_ins}]",
                        flush=True,
                    )
                except Exception as e:  # noqa: BLE001
                    print(f"  save FAIL: {type(e).__name__}: {e}", flush=True)
                finally:
                    db.close()
                # Jittered delay
                if i < len(urls_with_label):
                    j = delay * random.uniform(0.7, 1.3)
                    await asyncio.sleep(j)
        finally:
            await context.close()
            await browser.close()
    print(
        f"\n=== TOTAL: fetched={total_fetched} ins={total_ins} upd={total_upd} ===",
        flush=True,
    )
    return total_fetched


async def main() -> None:
    # Diagnostic — show env vars used:
    db_url = os.environ.get("DATABASE_URL", "<NOT SET>")
    # Mask password for logging
    if "://" in db_url and "@" in db_url:
        before_at, after_at = db_url.rsplit("@", 1)
        scheme_user = before_at.split("://", 1)
        if len(scheme_user) == 2 and ":" in scheme_user[1]:
            user = scheme_user[1].split(":", 1)[0]
            masked = f"{scheme_user[0]}://{user}:***@{after_at}"
        else:
            masked = db_url[:50] + "..."
    else:
        masked = db_url
    print(f"DATABASE_URL: {masked}", flush=True)
    print(f"ENVIRONMENT: {os.environ.get('ENVIRONMENT', '<default>')}", flush=True)

    p = argparse.ArgumentParser()
    p.add_argument("--mode", choices=["citywide", "byrooms"], default="citywide")
    p.add_argument("--pages", type=int, default=100, help="pages per query")
    p.add_argument("--delay", type=float, default=7, help="sec между requests")
    p.add_argument("--headless", action="store_true", default=True)
    p.add_argument("--no-headless", dest="headless", action="store_false")
    p.add_argument(
        "--start-cat",
        type=int,
        default=1,
        help="byrooms: с какой категории начать (1-based). 1=студии, 2=1-комн, "
             "3=2-комн, 4=3-комн, 5=4-комн, 6=5+комн, 7=своб.планир.",
    )
    args = p.parse_args()

    urls: list[tuple[str, str]] = []
    if args.mode == "citywide":
        for page in range(1, args.pages + 1):
            urls.append((f"citywide page={page}/{args.pages}", build_citywide_url(page)))
    else:  # byrooms
        cats = ROOM_SLUGS[args.start_cat - 1:]
        if not cats:
            print(f"ERROR: --start-cat={args.start_cat} > {len(ROOM_SLUGS)} cats", flush=True)
            return
        print(f"byrooms: starting from cat #{args.start_cat} ({cats[0][0]!r}), "
              f"{len(cats)} cats total", flush=True)
        for cat_name, slug in cats:
            for page in range(1, args.pages + 1):
                urls.append((f"{cat_name} page={page}/{args.pages}", build_rooms_url(slug, page)))

    print(f"Mode: {args.mode}, pages: {args.pages}, delay: {args.delay}s, total URLs: {len(urls)}")
    print(f"Estimated time: {len(urls) * args.delay / 60:.0f} min (без challenge)")
    t0 = time.perf_counter()
    await sweep_urls(urls, args.delay, args.headless)
    print(f"\nDONE in {(time.perf_counter() - t0) / 60:.1f} min")


if __name__ == "__main__":
    asyncio.run(main())