"""Pull every available Objective report combination for Ekb.

The API has 4 reports = ReportType × ReportName:
  Сводные / Корпуса        — already pulled in 05_*
  Сводные / Лоты           — probe
  Поквартирные / Корпуса   — probe
  Поквартирные / Лоты      — full per-flat data (most valuable)

Earlier server graph said 2 of 4 returned HTTP 500. We probe again,
then bulk-pull whatever returns 200.
"""
import sqlite3, pathlib, requests, time, json, datetime as dt, sys

DB = pathlib.Path(__file__).parent / "analysis.db"
RAW = pathlib.Path(__file__).parent / "cache" / "objective_raw"
RAW.mkdir(parents=True, exist_ok=True)

API = "https://api.objctv.ru"
KEY = "623f6a57-0179-434b-8202-259525bdc77c"
GROUP = "Екатеринбург"
TODAY = dt.date.today()
SD = (TODAY.replace(day=1) - dt.timedelta(days=365)).strftime("%Y.%m.%d")
ED = TODAY.strftime("%Y.%m.%d")

EXTRA_SCHEMA = """
-- per-flat snapshot (Поквартирные/Лоты)
CREATE TABLE IF NOT EXISTS objective_lots (
    lot_id          INTEGER PRIMARY KEY,    -- field "Id"
    project_id      INTEGER,
    project         TEXT,
    developer       TEXT,
    city            TEXT,
    district        TEXT,
    corpus          TEXT,
    address         TEXT,
    obj_class       TEXT,
    sales_start     TEXT,
    plan_date       TEXT,
    fact_date       TEXT,
    readiness_pct   REAL,
    construction_stage TEXT,
    finish_type     TEXT,
    section         TEXT,
    floor           INTEGER,
    lot_num         TEXT,
    room_kind       TEXT,
    status          TEXT,
    sold            TEXT,
    rooms_dev       TEXT,
    rooms_pd        TEXT,
    rooms_obj       TEXT,
    area_dev        REAL,
    area_pd         REAL,
    budget_rub      REAL,
    price_per_m2    REAL,
    price_method    TEXT,
    price_set_date  TEXT,
    price_actual_date TEXT,
    offer_price     REAL,
    delta_price_rub REAL,
    delta_price_pct REAL,
    contract_date   TEXT,
    register_date   TEXT,
    deal_type       TEXT,
    buyer_type      TEXT,
    register_num    TEXT,
    encumbrance     TEXT,
    bank            TEXT,
    encumbrance_start TEXT,
    egrn_actual_date TEXT,
    fetched_at      TEXT DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS lots_proj  ON objective_lots(project);
CREATE INDEX IF NOT EXISTS lots_dist  ON objective_lots(district);
CREATE INDEX IF NOT EXISTS lots_bank  ON objective_lots(bank);
CREATE INDEX IF NOT EXISTS lots_status ON objective_lots(status);
"""

# Mapping for per-flat (Поквартирные/Лоты)
LOTS_MAP = {
    "lot_id":"Id","project_id":"Id проекта","project":"Проект","developer":"Девелопер",
    "city":"Город","district":"Район","corpus":"Корпус","address":"Адрес",
    "obj_class":"Класс","sales_start":"Старт продаж",
    "plan_date":"Планируемая дата ввода","fact_date":"Фактическая дата ввода",
    "readiness_pct":"Готовность","construction_stage":"Стадия строительства",
    "finish_type":"Отделка по корпусу","section":"Секция","floor":"Этаж",
    "lot_num":"Номер лота","room_kind":"Вид помещения","status":"Статус",
    "sold":"Продано","rooms_dev":"Количество комнат(Сайт девелопера)",
    "rooms_pd":"Количество комнат(ПД)","rooms_obj":"Количество комнат(Данные объектива)",
    "area_dev":"Площадь, м2(Сайт девелопера)","area_pd":"Площадь, м2(ПД)",
    "budget_rub":"Расчетный бюджет лота, Р","price_per_m2":"Цена за м2, Р",
    "price_method":"Способ определения цены","price_set_date":"Дата установки цены",
    "price_actual_date":"Дата актуальности цены","offer_price":"Цена предложения, Р",
    "delta_price_rub":"Дельта цена, Р","delta_price_pct":"Дельта цена, %",
    "contract_date":"Дата договора","register_date":"Дата регистрации",
    "deal_type":"Тип сделки","buyer_type":"Тип покупателя","register_num":"Номер регистрации",
    "encumbrance":"Тип обременения","bank":"Банк","encumbrance_start":"Дата начала обременения",
    "egrn_actual_date":"Дата актуальности данных из ЕГРН",
}


def get_token():
    r = requests.get(f"{API}/Users/User/GetToken", params={"apiKey": KEY}, timeout=30)
    r.raise_for_status()
    return r.json()["token"]


def fetch(token, params, attempts=5):
    """Fetch with backoff for 429/5xx."""
    last = None
    for i in range(attempts):
        r = requests.get(f"{API}/v2/Report/GetReport",
            params=params,
            headers={"Authorization": f"Bearer {token}", "Accept-Encoding": "br"},
            timeout=180)
        if r.ok:
            return r
        last = (r.status_code, r.text[:200])
        # If 401 — refresh token
        if r.status_code == 401:
            token = get_token()
            continue
        wait = int(r.headers.get("Retry-After") or (10 * (2 ** i)))
        print(f"    HTTP {r.status_code} · ждём {wait}с (попытка {i+1}/{attempts})")
        time.sleep(min(wait, 120))
    raise RuntimeError(f"failed after {attempts}: {last}")


def normalize_pct(s):
    if s is None: return None
    s = str(s).strip()
    if s.endswith("%"): s = s[:-1]
    try: return float(s)
    except: return None


def parse_dec(v):
    if v in (None, ""): return None
    if isinstance(v, (int, float)): return v
    try: return float(str(v).replace(" ","").replace(",","."))
    except: return None


def main():
    conn = sqlite3.connect(DB)
    conn.executescript(EXTRA_SCHEMA)

    token = get_token()
    print(f"Group={GROUP}  window={SD}..{ED}")

    # -------- probe all four (without ComplexName, then with) --------
    print("\n=== Probe all four reports ===")
    for rt in ("Сводные", "Поквартирные"):
        for rn in ("Корпуса", "Лоты"):
            base = {"Page":"Отчеты","ReportSection":"Объединенные данные",
                    "ReportType":rt,"ReportName":rn,"GroupName":GROUP,
                    "UseDdu":"true","UseDkp":"true"}
            # Сводные wants StartDate/EndDate
            if rt == "Сводные":
                base["StartDate"] = SD; base["EndDate"] = ED
            try:
                r = fetch(token, base, attempts=2)
                rows = r.json().get("result", [])
                rows_n = len(rows) if isinstance(rows, list) else "?"
                print(f"  {rt}/{rn:<7} (whole Ekb): rows={rows_n}, bytes={len(r.content)}")
                ts = dt.datetime.now().strftime("%Y%m%dT%H%M%S")
                fname = RAW / f"{rt}_{rn}_whole_{ts}.json"
                fname.write_text(json.dumps(rows, ensure_ascii=False))
                if isinstance(rows, list) and rows:
                    print(f"    sample keys ({len(rows[0])}): {list(rows[0].keys())[:8]}")
            except Exception as e:
                print(f"  {rt}/{rn} FAIL: {e}")
                # try with ComplexName fallback
                try:
                    base2 = dict(base, ComplexName="Парк Культуры")
                    r = fetch(token, base2, attempts=2)
                    rows = r.json().get("result", [])
                    rows_n = len(rows) if isinstance(rows, list) else "?"
                    print(f"    +ComplexName=Парк Культуры → rows={rows_n}")
                    ts = dt.datetime.now().strftime("%Y%m%dT%H%M%S")
                    (RAW / f"{rt}_{rn}_pk_{ts}.json").write_text(json.dumps(rows, ensure_ascii=False))
                except Exception as e2:
                    print(f"    +ComplexName FAIL: {e2}")
            time.sleep(15)  # be nice — Objective rate-limits aggressively

    # -------- bulk-pull Поквартирные/Лоты (the most valuable per-flat data) --------
    # If whole-Ekb works, prefer that. Otherwise iterate over distinct projects from corp_month.
    print("\n=== Bulk pull Поквартирные/Лоты ===")
    pf_params = {"Page":"Отчеты","ReportSection":"Объединенные данные",
                 "ReportType":"Поквартирные","ReportName":"Лоты","GroupName":GROUP,
                 "UseDdu":"true","UseDkp":"true"}
    rows = None
    try:
        r = fetch(token, pf_params, attempts=3)
        rows = r.json().get("result", [])
        print(f"  whole Ekb: {len(rows)} flats, {len(r.content)/1024:.0f} KB")
    except Exception as e:
        print(f"  whole-Ekb fail: {e}; falling back to per-project loop")

    if rows is None:
        projects = [p[0] for p in conn.execute(
            "SELECT DISTINCT project FROM objective_corp_month WHERE project IS NOT NULL ORDER BY project"
        ).fetchall()]
        print(f"  iterating {len(projects)} projects")
        rows = []
        for i, proj in enumerate(projects, 1):
            try:
                r = fetch(token, dict(pf_params, ComplexName=proj))
                got = r.json().get("result", [])
                rows.extend(got)
                print(f"    [{i}/{len(projects)}] {proj}: +{len(got)}")
            except Exception as e:
                print(f"    [{i}/{len(projects)}] {proj}: FAIL {e}")
            time.sleep(10)

    if rows:
        ts = dt.datetime.now().strftime("%Y%m%dT%H%M%S")
        (RAW / f"Поквартирные_Лоты_{ts}.json").write_text(json.dumps(rows, ensure_ascii=False))
        # Insert
        conn.execute("DELETE FROM objective_lots")
        ins = 0; skip = 0
        for r in rows:
            try:
                vals = {k: r.get(v) for k, v in LOTS_MAP.items()}
                vals["readiness_pct"]   = normalize_pct(vals["readiness_pct"])
                for f in ("area_dev","area_pd","budget_rub","price_per_m2","offer_price",
                          "delta_price_rub","delta_price_pct","floor","lot_id","project_id"):
                    vals[f] = parse_dec(vals[f])
                cols = list(vals.keys())
                conn.execute(f"INSERT OR REPLACE INTO objective_lots({','.join(cols)}) VALUES ({','.join(['?']*len(cols))})",
                             [vals[c] for c in cols])
                ins += 1
            except Exception:
                skip += 1
        conn.commit()
        print(f"  inserted: {ins}, skipped: {skip}")

        # Sanity stats
        n = conn.execute("SELECT count(*) FROM objective_lots").fetchone()[0]
        print(f"\n  Total flats stored: {n}")
        for line in conn.execute("""SELECT status, count(*) FROM objective_lots
                                    GROUP BY 1 ORDER BY 2 DESC""").fetchall():
            print(f"    status={line[0]:<25} {line[1]}")

        print("\n  Top banks (mortgage encumbrance):")
        for line in conn.execute("""SELECT bank, count(*) FROM objective_lots
                                    WHERE bank IS NOT NULL AND bank!=''
                                    GROUP BY 1 ORDER BY 2 DESC LIMIT 10""").fetchall():
            print(f"    {line[0]:<35} {line[1]}")

        print("\n  By district (priced lots):")
        for line in conn.execute("""SELECT district, count(*) total,
                                           SUM(CASE WHEN price_per_m2>0 THEN 1 ELSE 0 END) priced,
                                           ROUND(AVG(price_per_m2)/1000,1) avg_kp
                                    FROM objective_lots
                                    WHERE district IS NOT NULL
                                    GROUP BY 1 ORDER BY 2 DESC LIMIT 12""").fetchall():
            print(f"    {line[0]:<22} total={line[1]:>5} priced={line[2]:>5} avg={line[3]} тыс/м²")

    conn.close()
    print("\nDone.")

if __name__ == "__main__":
    main()
