"""Yandex.Недвижимость ЕКБ vtorichka sweep — runs locally через residential IP.

Why local: Yandex anti-bot (captcha redirect) включается быстрее на datacenter
IP + httpx fingerprint. Запуск с residential IP + Chrome120 TLS impersonation
(curl_cffi) + browser session cookies — проходит. Скрипт пишет в prod tradein
DB через SSH tunnel (localhost:15433 → tradein-postgres docker container).

Modes:
  --mode citywide  (default): один общий /vtorichniy-rynok/ sweep, ~25 страниц
                              × 23 cards = ~575 уникальных. Yandex SERP лимит.
  --mode combos:    итерация 5 rooms × 6 price ranges = 30 combos × до 25 pages.
                    DB-level дедуп → coverage близко к total (~4000+ vtorichka
                    ЕКБ). ~75 min полный прогон.

Setup:
  ssh -f -N -L 15433:172.20.0.2:5432 gendesign   # tradein-postgres tunnel
  cd tradein-mvp/backend
  export DATABASE_URL='postgresql+psycopg://tradein:<PASS>@localhost:15433/tradein'
  export ENVIRONMENT='local'
  export GLITCHTIP_DSN=''

Usage:
  uv run python ../scripts/local-sweep-ekb-yandex.py                       # citywide
  uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos          # full ~4000
  uv run python ../scripts/local-sweep-ekb-yandex.py --mode combos \\
      --rooms 1,2 --price-ranges 5-7,7-10                                   # subset
  uv run python ../scripts/local-sweep-ekb-yandex.py --pages 1              # smoke
"""
from __future__ import annotations

# stdlib
import argparse
import asyncio
import json
import random
import sys
import time
import types
from pathlib import Path
from urllib.parse import urlencode

# Auto-inject backend/ в sys.path чтобы найти app.* package
_BACKEND_DIR = Path(__file__).resolve().parent.parent / "backend"
if str(_BACKEND_DIR) not in sys.path:
    sys.path.insert(0, str(_BACKEND_DIR))

# Stub weasyprint (Windows без GTK ломает import chain через trade_in.py)
for _m in (
    "weasyprint",
    "weasyprint.text",
    "weasyprint.text.ffi",
    "weasyprint.css",
    "weasyprint.css.targets",
):
    sys.modules[_m] = types.ModuleType(_m)
sys.modules["weasyprint"].HTML = type("HTML", (), {"__init__": lambda self, *a, **kw: None})
sys.modules["weasyprint"].CSS = type("CSS", (), {"__init__": lambda self, *a, **kw: None})

# app imports после stubs
from curl_cffi.requests import AsyncSession  # noqa: E402

from app.core.db import SessionLocal  # noqa: E402
from app.services.scrapers.base import save_listings  # noqa: E402
from app.services.scrapers.yandex_realty import YandexRealtyScraper  # noqa: E402

DEFAULT_CITY = "ekaterinburg"
DEFAULT_COOKIES_FILE = Path(__file__).resolve().parent / ".yandex-cookies.json"

# Path-сегменты Yandex для room buckets (после канонизации). Ключ — slug для
# --rooms CLI; значение — segment в URL.
ROOM_PATH: dict[str, str] = {
    "studio": "studiya",
    "1": "odnokomnatnaya",
    "2": "dvuhkomnatnaya",
    "3": "trehkomnatnaya",
    "4+": "4-i-bolee",
}

# Price ranges (price_min, price_max) в рублях. None — open-ended boundary.
PRICE_RANGES: list[tuple[int | None, int | None]] = [
    (None, 5_000_000),
    (5_000_000, 7_000_000),
    (7_000_000, 10_000_000),
    (10_000_000, 15_000_000),
    (15_000_000, 25_000_000),
    (25_000_000, None),
]

_CAPTCHA_MARKERS = ("/showcaptcha", "smartcaptcha", "checkboxcaptcha")


def _load_cookies(path: Path) -> dict[str, str]:
    if not path.exists():
        return {}
    raw = json.loads(path.read_text(encoding="utf-8"))
    return {item["name"]: item["value"] for item in raw if "name" in item and "value" in item}


def _is_captcha(html: str) -> bool:
    head = html[:5000].lower()
    return any(m in head for m in _CAPTCHA_MARKERS) or "докажите, что вы не робот" in head


def _build_url(
    city: str,
    page: int = 0,
    rooms: str | None = None,
    price_min: int | None = None,
    price_max: int | None = None,
) -> str:
    """Yandex SERP URL. Если rooms задан — path-based бакет; иначе общий vtorichka.

    newFlat=NO_DEAL принудительно отфильтровывает новостройки (нужно при
    combo, т.к. Yandex canonicalize иногда дропает /vtorichniy-rynok/).
    """
    if rooms:
        path = f"/{city}/kupit/kvartira/{ROOM_PATH[rooms]}/vtorichniy-rynok/"
    else:
        path = f"/{city}/kupit/kvartira/vtorichniy-rynok/"
    base = f"https://realty.yandex.ru{path}"

    params: dict[str, str | int] = {}
    if price_min is not None:
        params["priceMin"] = price_min
    if price_max is not None:
        params["priceMax"] = price_max
    if price_min is not None or price_max is not None:
        # Гарантия vtorichka — Yandex иногда дропает /vtorichniy-rynok/ при canonicalize
        params["newFlat"] = "NO_DEAL"
    if page > 0:
        params["page"] = page

    if params:
        return f"{base}?{urlencode(params)}"
    return base


def _combo_label(rooms: str | None, price_min: int | None, price_max: int | None) -> str:
    r = rooms or "all-rooms"
    if price_min is None and price_max is None:
        return f"{r}/any-price"
    lo = f"{price_min // 1_000_000}M" if price_min else "0"
    hi = f"{price_max // 1_000_000}M" if price_max else "inf"
    return f"{r}/{lo}-{hi}"


async def sweep_one(
    session: AsyncSession,
    parser: YandexRealtyScraper,
    city: str,
    pages: int,
    delay: float,
    rooms: str | None = None,
    price_min: int | None = None,
    price_max: int | None = None,
) -> tuple[int, int, int]:
    """Один scan по фиксированному (rooms, price) — пагинация до pages или empty.

    Returns: (fetched, inserted, updated)
    """
    label = _combo_label(rooms, price_min, price_max)
    total = cum_ins = cum_upd = 0
    for page in range(pages):
        url = _build_url(city, page=page, rooms=rooms, price_min=price_min, price_max=price_max)
        try:
            resp = await session.get(url, timeout=20)
        except Exception as e:  # noqa: BLE001
            print(f"    [{label}] page={page} FAIL fetch: {type(e).__name__}: {e}", flush=True)
            await asyncio.sleep(delay)
            continue
        if resp.status_code != 200:
            print(f"    [{label}] page={page} HTTP {resp.status_code} — stop combo", flush=True)
            break
        if _is_captcha(resp.text):
            print(
                f"    [{label}] page={page} ⚠️ CAPTCHA — stop combo. "
                f"Подожди 30-60min или refresh .yandex-cookies.json",
                flush=True,
            )
            raise RuntimeError("captcha")
        # NBSP → space: yandex_helpers.parse_rub() багует на \xa0 в group(1) → price None
        html = resp.text.replace("\xa0", " ")
        lots = parser._parse_html(html, page=page)
        if not lots:
            print(f"    [{label}] page={page}: 0 lots — end of pagination, stop", flush=True)
            break
        db = SessionLocal()
        try:
            ins, upd = save_listings(db, lots)
            cum_ins += ins
            cum_upd += upd
            total += len(lots)
            print(
                f"    [{label}] page={page + 1}/{pages}: {len(lots)} lots → "
                f"ins={ins} upd={upd} [combo: {total}/{cum_ins}/{cum_upd}]",
                flush=True,
            )
        except Exception as e:  # noqa: BLE001
            print(f"    [{label}] page={page} FAIL save: {type(e).__name__}: {e}", flush=True)
        finally:
            db.close()
        if page + 1 < pages:
            await asyncio.sleep(delay * random.uniform(0.85, 1.3))
    return total, cum_ins, cum_upd


async def phase_citywide(pages: int, delay: float, city: str, cookies: dict[str, str]) -> None:
    print(
        f"\n=== PHASE CITYWIDE: Yandex vtorichka (city={city}, {pages} pages, "
        f"delay {delay}s) ===\n",
        flush=True,
    )
    parser = YandexRealtyScraper(city=city)
    async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
        total, ins, upd = await sweep_one(session, parser, city, pages, delay)
    print(f"  CITYWIDE TOTAL: fetched={total} ins={ins} upd={upd}", flush=True)


async def phase_combos(
    pages: int,
    delay: float,
    city: str,
    cookies: dict[str, str],
    rooms_list: list[str],
    price_ranges: list[tuple[int | None, int | None]],
    combo_delay: float,
) -> None:
    combos = [(r, lo, hi) for r in rooms_list for lo, hi in price_ranges]
    print(
        f"\n=== PHASE COMBOS: {len(combos)} combos × ≤{pages} pages "
        f"(city={city}, delay {delay}s, combo-delay {combo_delay}s) ===",
        flush=True,
    )
    parser = YandexRealtyScraper(city=city)
    grand_fetched = grand_ins = grand_upd = 0
    async with AsyncSession(impersonate="chrome120", cookies=cookies or {}) as session:
        for i, (rooms, lo, hi) in enumerate(combos, 1):
            label = _combo_label(rooms, lo, hi)
            print(f"\n  [{i}/{len(combos)}] combo={label}", flush=True)
            try:
                f, ins, upd = await sweep_one(
                    session, parser, city, pages, delay,
                    rooms=rooms, price_min=lo, price_max=hi,
                )
            except RuntimeError as e:
                if str(e) == "captcha":
                    print("  ⚠️ stopping combo run — captcha", flush=True)
                    break
                raise
            grand_fetched += f
            grand_ins += ins
            grand_upd += upd
            print(
                f"  [{i}/{len(combos)}] {label} done: f={f} i={ins} u={upd} "
                f"[grand: f={grand_fetched} i={grand_ins} u={grand_upd}]",
                flush=True,
            )
            if i < len(combos):
                await asyncio.sleep(combo_delay * random.uniform(0.85, 1.3))
    print(
        f"\n  COMBOS GRAND TOTAL: fetched={grand_fetched} "
        f"ins={grand_ins} upd={grand_upd}",
        flush=True,
    )


def _parse_rooms_arg(s: str) -> list[str]:
    if s == "all":
        return list(ROOM_PATH)
    out: list[str] = []
    for tok in s.split(","):
        tok = tok.strip()
        if tok not in ROOM_PATH:
            raise argparse.ArgumentTypeError(
                f"unknown room '{tok}'. Allowed: {sorted(ROOM_PATH)} or 'all'"
            )
        out.append(tok)
    return out


def _parse_price_ranges_arg(s: str) -> list[tuple[int | None, int | None]]:
    if s == "all":
        return list(PRICE_RANGES)
    out: list[tuple[int | None, int | None]] = []
    for tok in s.split(","):
        tok = tok.strip()
        if "-" not in tok:
            raise argparse.ArgumentTypeError(f"range '{tok}' missing '-'")
        lo_s, hi_s = tok.split("-", 1)
        lo = int(float(lo_s) * 1_000_000) if lo_s else None
        hi = int(float(hi_s) * 1_000_000) if hi_s else None
        out.append((lo, hi))
    return out


async def main() -> None:
    p = argparse.ArgumentParser(description="Yandex.Недвижимость ЕКБ sweep")
    p.add_argument(
        "--mode",
        choices=["citywide", "combos"],
        default="citywide",
        help="citywide = один общий vtorichka sweep (~575 уникальных). "
        "combos = rooms × price = ~4000+",
    )
    p.add_argument(
        "--pages",
        type=int,
        default=25,
        help="max pages per sweep (Yandex SERP cap ≈ 25; stops earlier on empty)",
    )
    p.add_argument(
        "--delay",
        type=float,
        default=15,
        help="sec между HTTP requests внутри combo (default 15, residential-safe)",
    )
    p.add_argument(
        "--combo-delay",
        type=float,
        default=20,
        help="sec между combos (default 20, дополнительный jitter combos mode)",
    )
    p.add_argument(
        "--rooms",
        type=_parse_rooms_arg,
        default="all",
        help=f"combos: comma-list из {sorted(ROOM_PATH)} или 'all' (default)",
    )
    p.add_argument(
        "--price-ranges",
        type=_parse_price_ranges_arg,
        default="all",
        help="combos: comma-list 'lo-hi' в млн руб ('-5,5-7,7-10,10-15,15-25,25-') "
        "или 'all' (default). Empty side = open-ended.",
    )
    p.add_argument(
        "--city",
        type=str,
        default=DEFAULT_CITY,
        help=f"Yandex city slug (default '{DEFAULT_CITY}')",
    )
    p.add_argument(
        "--cookies-file",
        type=Path,
        default=DEFAULT_COOKIES_FILE,
        help=f"Browser cookies JSON (default '{DEFAULT_COOKIES_FILE.name}'). "
        f"Empty/missing → start без cookies (риск captcha).",
    )
    args = p.parse_args()

    cookies = _load_cookies(args.cookies_file)
    if cookies:
        print(f"loaded {len(cookies)} cookies from {args.cookies_file}", flush=True)
    else:
        print(
            f"⚠️ no cookies loaded ({args.cookies_file}, exists={args.cookies_file.exists()}). "
            f"Yandex anti-bot risk.",
            flush=True,
        )

    t0 = time.perf_counter()
    if args.mode == "citywide":
        await phase_citywide(args.pages, args.delay, args.city, cookies)
    else:
        await phase_combos(
            args.pages, args.delay, args.city, cookies,
            rooms_list=args.rooms,
            price_ranges=args.price_ranges,
            combo_delay=args.combo_delay,
        )
    elapsed = time.perf_counter() - t0
    print(f"\n=== ALL DONE in {elapsed / 60:.1f} min ===", flush=True)


if __name__ == "__main__":
    asyncio.run(main())