"""Loads DOM.RF extra snapshots (XLSX downloads + PDF-extracted data) into Postgres.

Inputs in C:\\Users\\user\\Downloads\\:
  - developer_rf_26-04-2026.xlsx        → domrf_developers_full
  - escrow_rf_26-04-2026.xlsx           → domrf_escrow_banks
  - guaranty_26-04-2026.xlsx            → domrf_guaranty_regions
  - housing_info_26-04-2026.pdf         → domrf_housing_summary, domrf_planned_commissioning  (hardcoded extract)
  - sold_out_info_26-04-2026.pdf        → domrf_sold_out, ..._by_year, ..._by_progress         (hardcoded extract)
  - mortgage_rates_19-04-2026.pdf       → domrf_mortgage_rates                                  (hardcoded extract)

Usage:
    PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... python data/sql/21_load_domrf_extras.py
"""
import os, sys, subprocess
import openpyxl

DOWNLOADS = r'C:\Users\user\Downloads'
HERE = os.path.dirname(os.path.abspath(__file__))
PG_PORT = os.environ.get('PG_PORT', '5432')
PG_USER = os.environ.get('PG_USER', 'gendesign')
PG_DB   = os.environ.get('PG_DB',   'gendesign')
PG_PASS = os.environ.get('PGPASSWORD', '')


def psql(sql):
    cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
           'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT,
           '-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet']
    res = subprocess.run(cmd, input=sql, capture_output=True, text=True, encoding='utf-8')
    if res.returncode != 0:
        print('psql FAILED:', res.stderr[-2000:], file=sys.stderr)
        raise SystemExit(res.returncode)
    return res.stdout


def esc(s):
    if s is None: return 'NULL'
    return "'" + str(s).replace("'", "''") + "'"

def num(v):
    if v is None or v == '-' or v == '': return 'NULL'
    try: return str(float(v))
    except (ValueError, TypeError): return 'NULL'

def integer(v):
    if v is None or v == '-' or v == '': return 'NULL'
    try: return str(int(float(v)))
    except (ValueError, TypeError): return 'NULL'


def main():
    # 0. Apply schema
    schema = open(os.path.join(HERE, '20_schema_domrf_extras.sql'), encoding='utf-8').read()
    psql(schema)
    print('schema applied')

    SNAP_APR26 = '2026-04-26'
    SNAP_MORTGAGE = '2026-04-19'
    SNAP_SOLD = '2026-04-11'  # отчётная дата на самом PDF

    # ─────────── 1. developer_rf.xlsx → domrf_developers_full ───────────
    wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'developer_rf_26-04-2026.xlsx'),
                                read_only=True, data_only=True)
    rows = list(wb.active.iter_rows(values_only=True))
    wb.close()
    # rows[3] = header; rows[4] = numeric column index; rows[5] = 'Итого'; rows[6+] = data
    seen = {}
    for r in rows[6:]:
        name = r[0]
        if not name or not isinstance(name, str) or name.strip() == '': continue
        # Deduplicate: keep entry with highest area (xlsx может содержать дубли названий)
        key = name.strip()
        prev = seen.get(key)
        cur_area = r[1] if r[1] is not None else 0
        if prev is None or (prev[1] or 0) < cur_area:
            seen[key] = r
    parts = [f"('{SNAP_APR26}', {esc(k)}, {num(r[1])}, {integer(r[2])}, "
             f"{integer(r[3])}, {integer(r[4])}, {num(r[5])})"
             for k, r in seen.items()]
    BATCH = 500
    for i in range(0, len(parts), BATCH):
        psql(f"""
            INSERT INTO domrf_developers_full (snapshot_date, developer_name, area_thousand_sqm,
                permits_count, houses_count, flats_count, market_share_pct)
            VALUES {','.join(parts[i:i+BATCH])}
            ON CONFLICT (snapshot_date, developer_name) DO UPDATE SET
                area_thousand_sqm = EXCLUDED.area_thousand_sqm,
                permits_count = EXCLUDED.permits_count,
                houses_count = EXCLUDED.houses_count,
                flats_count = EXCLUDED.flats_count,
                market_share_pct = EXCLUDED.market_share_pct;
        """)
    print(f'developers_full: {len(parts)} (deduped from {len(rows)-6})')

    # ─────────── 2. escrow_rf.xlsx → domrf_escrow_banks ───────────
    wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'escrow_rf_26-04-2026.xlsx'),
                                read_only=True, data_only=True)
    rows = list(wb.active.iter_rows(values_only=True))
    wb.close()
    parts = []
    for r in rows[6:]:
        name = r[0]
        if not name or not isinstance(name, str) or name.strip() == '' or name == 'Итого': continue
        parts.append(f"('{SNAP_APR26}', {esc(name.strip())}, {integer(r[1])}, {integer(r[2])}, "
                     f"{num(r[3])}, {integer(r[4])}, {integer(r[5])})")
    psql(f"""
        INSERT INTO domrf_escrow_banks (snapshot_date, bank_name, houses_count, permits_count,
            area_thousand_sqm, developers_count, regions_count)
        VALUES {','.join(parts)}
        ON CONFLICT (snapshot_date, bank_name) DO UPDATE SET
            houses_count = EXCLUDED.houses_count,
            permits_count = EXCLUDED.permits_count,
            area_thousand_sqm = EXCLUDED.area_thousand_sqm,
            developers_count = EXCLUDED.developers_count,
            regions_count = EXCLUDED.regions_count;
    """)
    print(f'escrow_banks: {len(parts)}')

    # ─────────── 3. guaranty.xlsx → domrf_guaranty_regions ───────────
    wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'guaranty_26-04-2026.xlsx'),
                                read_only=True, data_only=True)
    rows = list(wb.active.iter_rows(values_only=True))
    wb.close()
    # rows[6] = РФ; rows[7..] = ФО + регионы
    FO_NAMES = {'Центральный ФО', 'Северо-Западный ФО', 'Южный ФО', 'Северо-Кавказский ФО',
                'Приволжский ФО', 'Уральский ФО', 'Сибирский ФО', 'Дальневосточный ФО',
                'Новые субъекты Российской Федерации'}
    parts = []
    for r in rows[6:]:
        name = r[0]
        if not name or not isinstance(name, str) or name.strip() == '': continue
        name = name.strip()
        if name == 'Российская Федерация':       level = 'rf'
        elif name in FO_NAMES:                   level = 'fo'
        else:                                    level = 'region'
        parts.append(f"('{SNAP_APR26}', {esc(name)}, '{level}', "
                     f"{integer(r[1])}, {num(r[2])}, {integer(r[3])}, "
                     f"{integer(r[4])}, {num(r[5])}, {integer(r[6])}, {num(r[7])}, "
                     f"{integer(r[8])}, {num(r[9])}, {integer(r[10])}, "
                     f"{integer(r[11])}, {num(r[12])}, {integer(r[13])})")
    BATCH = 100
    for i in range(0, len(parts), BATCH):
        psql(f"""
            INSERT INTO domrf_guaranty_regions (snapshot_date, territory_name, territory_level,
                total_permits, total_area_th_sqm, total_developers,
                fz214_permits, fz214_area_th_sqm, fz214_developers, fz214_pct_of_total,
                escrow_permits, escrow_area_th_sqm, escrow_developers,
                pp480_permits, pp480_area_th_sqm, pp480_developers)
            VALUES {','.join(parts[i:i+BATCH])}
            ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
                total_permits = EXCLUDED.total_permits,
                total_area_th_sqm = EXCLUDED.total_area_th_sqm,
                total_developers = EXCLUDED.total_developers,
                fz214_permits = EXCLUDED.fz214_permits,
                fz214_area_th_sqm = EXCLUDED.fz214_area_th_sqm,
                fz214_developers = EXCLUDED.fz214_developers,
                fz214_pct_of_total = EXCLUDED.fz214_pct_of_total,
                escrow_permits = EXCLUDED.escrow_permits,
                escrow_area_th_sqm = EXCLUDED.escrow_area_th_sqm,
                escrow_developers = EXCLUDED.escrow_developers,
                pp480_permits = EXCLUDED.pp480_permits,
                pp480_area_th_sqm = EXCLUDED.pp480_area_th_sqm,
                pp480_developers = EXCLUDED.pp480_developers;
        """)
    print(f'guaranty_regions: {len(parts)}')

    # ─────────── 4. housing_info.pdf → domrf_housing_summary + planned_commissioning ───────────
    # extracted manually from PDF
    housing_summary = [
        # (territory, level, mechanism, devs, permits, decls, houses, area, flats_th)
        ('Российская Федерация', 'rf', 'all',           4336, 7257, 7447, 11510, 119234, 2424),
        ('Российская Федерация', 'rf', 'escrow',        4285, 7170, 7359, 11385, 117670, 2394),
        ('Российская Федерация', 'rf', 'fund',          42,   61,   62,   85,    1104,   22),
        ('Российская Федерация', 'rf', 'no_attraction', 15,   28,   28,   40,    460,    8),
    ]
    # top-10 regions (escrow / fund / no_attraction / total)  area_th_sqm
    region_breakdown = [
        # (region, escrow, fund, no_attr, total)
        ('Город Москва',           15670, 364, 286, 16320),
        ('Краснодарский край',     8179,  24,  4,   8206),
        ('Московская область',     7905,  81,  None, 7986),
        ('Свердловская область',   5637,  17,  None, 5654),
        ('Город Санкт-Петербург',  5258,  None, None, 5258),
        ('Ростовская область',     4926,  None, None, 4926),
        ('Ленинградская область',  4640,  60,  None, 4700),
        ('Новосибирская область',  3580,  107, None, 3687),
        ('Республика Башкортостан',3583,  None, None, 3583),
        ('Тюменская область',      3513,  None, 11,  3524),
    ]
    city_breakdown = [
        # (city, escrow, fund, no_attr, total)
        ('Москва',           15670, 364, 286, 16320),
        ('Краснодар',        5791,  None, None, 5791),
        ('Екатеринбург',     5294,  17,  None, 5310),
        ('Санкт-Петербург',  5258,  None, None, 5258),
        ('Ростов-на-Дону',   3605,  None, None, 3605),
        ('Тюмень',           3204,  None, 11,  3215),
        ('Новосибирск',      2755,  104, None, 2859),
        ('Уфа',              2786,  None, None, 2786),
        ('Казань',           2205,  None, None, 2205),
        ('Владивосток',      2134,  None, None, 2134),
    ]
    parts = []
    for (terr, level, mech, devs, permits, decls, houses, area, flats) in housing_summary:
        parts.append(f"('{SNAP_APR26}', {esc(terr)}, '{level}', '{mech}', {integer(devs)}, "
                     f"{integer(permits)}, {integer(decls)}, {integer(houses)}, {num(area)}, {num(flats)})")
    for breakdown, level in [(region_breakdown, 'region'), (city_breakdown, 'city')]:
        for (terr, escrow, fund, no_attr, total) in breakdown:
            for mech, area in [('escrow', escrow), ('fund', fund),
                               ('no_attraction', no_attr), ('all', total)]:
                if area is None: continue
                parts.append(f"('{SNAP_APR26}', {esc(terr)}, '{level}', '{mech}', NULL, "
                             f"NULL, NULL, NULL, {num(area)}, NULL)")
    psql(f"""
        INSERT INTO domrf_housing_summary (snapshot_date, territory_name, territory_level,
            mechanism, developers_count, permits_count, declarations_count, houses_count,
            area_thousand_sqm, flats_thousand)
        VALUES {','.join(parts)}
        ON CONFLICT (snapshot_date, territory_name, mechanism) DO UPDATE SET
            developers_count = EXCLUDED.developers_count,
            permits_count = EXCLUDED.permits_count,
            declarations_count = EXCLUDED.declarations_count,
            houses_count = EXCLUDED.houses_count,
            area_thousand_sqm = EXCLUDED.area_thousand_sqm,
            flats_thousand = EXCLUDED.flats_thousand;
    """)
    print(f'housing_summary: {len(parts)}')

    # planned_commissioning (year, escrow, fund, no_attr, total)
    pc = [
        (2026, 37084, 685, 154, 37923),
        (2027, 34896, 246, 221, 35363),
        (2028, 26360, 113, 30,  26503),
        (2029, 10924, 19,  24,  10967),
        (2030, 4245,  13,  32,  4290),
        (2031, 1797,  28,  None, 1825),
        (2032, 1323,  None, None, 1323),
        (2033, 351,   None, None, 351),
        (2034, 234,   None, None, 234),
        (2035, 67,    None, None, 67),
    ]
    parts = [f"('{SNAP_APR26}', {y}, {num(e)}, {num(f)}, {num(n)}, {num(t)})"
             for (y, e, f, n, t) in pc]
    psql(f"""
        INSERT INTO domrf_planned_commissioning (snapshot_date, plan_year, escrow_th_sqm,
            fund_th_sqm, no_attraction_th_sqm, total_th_sqm)
        VALUES {','.join(parts)}
        ON CONFLICT (snapshot_date, plan_year) DO UPDATE SET
            escrow_th_sqm = EXCLUDED.escrow_th_sqm,
            fund_th_sqm = EXCLUDED.fund_th_sqm,
            no_attraction_th_sqm = EXCLUDED.no_attraction_th_sqm,
            total_th_sqm = EXCLUDED.total_th_sqm;
    """)
    print(f'planned_commissioning: {len(parts)}')

    # ─────────── 5. sold_out_info.pdf → 3 tables ───────────
    # RF total
    psql(f"""
        INSERT INTO domrf_sold_out (snapshot_date, territory_name, total_area_th_sqm,
            sold_area_th_sqm, unsold_area_th_sqm, not_open_area_th_sqm,
            sold_pct, unsold_pct, not_open_pct, avg_price_per_sqm, attracted_funds_mln_rub)
        VALUES ('{SNAP_SOLD}', 'Российская Федерация',
            118876, 36749, 53044, 29084, 31, 45, 24, 216316, 7949316)
        ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
            total_area_th_sqm = EXCLUDED.total_area_th_sqm,
            sold_area_th_sqm = EXCLUDED.sold_area_th_sqm,
            unsold_area_th_sqm = EXCLUDED.unsold_area_th_sqm,
            not_open_area_th_sqm = EXCLUDED.not_open_area_th_sqm,
            sold_pct = EXCLUDED.sold_pct,
            unsold_pct = EXCLUDED.unsold_pct,
            not_open_pct = EXCLUDED.not_open_pct,
            avg_price_per_sqm = EXCLUDED.avg_price_per_sqm,
            attracted_funds_mln_rub = EXCLUDED.attracted_funds_mln_rub;
    """)
    # Top-8 regions: (region, total, sold%, unsold%, not_open%)
    so_regions = [
        ('Город Москва',           16405, 47, 44, None),
        ('Краснодарский край',     8286,  20, 39, 41),
        ('Московская область',     7942,  37, 43, None),
        ('Свердловская область',   5686,  29, 54, None),
        ('Город Санкт-Петербург',  5336,  40, 44, None),
        ('Ростовская область',     4990,  None, 48, None),
        ('Ленинградская область',  4705,  None, 44, None),
        ('Тюменская область',      3573,  None, 50, None),
    ]
    parts = []
    for (r, total, sp, up, nop) in so_regions:
        parts.append(f"('{SNAP_SOLD}', {esc(r)}, {num(total)}, NULL, NULL, NULL, "
                     f"{num(sp)}, {num(up)}, {num(nop)}, NULL, NULL)")
    psql(f"""
        INSERT INTO domrf_sold_out (snapshot_date, territory_name, total_area_th_sqm,
            sold_area_th_sqm, unsold_area_th_sqm, not_open_area_th_sqm,
            sold_pct, unsold_pct, not_open_pct, avg_price_per_sqm, attracted_funds_mln_rub)
        VALUES {','.join(parts)}
        ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
            total_area_th_sqm = EXCLUDED.total_area_th_sqm,
            sold_pct = EXCLUDED.sold_pct,
            unsold_pct = EXCLUDED.unsold_pct,
            not_open_pct = EXCLUDED.not_open_pct;
    """)
    print(f'sold_out: 1 RF + {len(parts)} regions')

    # by_year — (year, total_th_sqm, pct, sold%, unsold%, not_open%)
    so_year = [
        ('2026',  40201, 34, 52, 38, 10),
        ('2027',  34880, 29, 30, 53, 17),
        ('2028',  25363, 21, 17, 52, 31),
        ('2029',  10359, 9,  None, 37, 54),
        ('2030',  4127,  3,  None, None, 71),
        ('2031+', 3946,  3,  None, None, 65),
    ]
    parts = [f"('{SNAP_SOLD}', '{y}', {num(t)}, {num(p)}, {num(s)}, {num(u)}, {num(n)})"
             for (y, t, p, s, u, n) in so_year]
    psql(f"""
        INSERT INTO domrf_sold_out_by_year (snapshot_date, plan_year, total_th_sqm,
            pct_of_total, sold_pct, unsold_pct, not_open_pct)
        VALUES {','.join(parts)}
        ON CONFLICT (snapshot_date, plan_year) DO UPDATE SET
            total_th_sqm = EXCLUDED.total_th_sqm,
            pct_of_total = EXCLUDED.pct_of_total,
            sold_pct = EXCLUDED.sold_pct,
            unsold_pct = EXCLUDED.unsold_pct,
            not_open_pct = EXCLUDED.not_open_pct;
    """)
    print(f'sold_out_by_year: {len(parts)}')

    # by_progress
    so_prog = [
        ('not_open', 29084, 24, None, 100),
        ('0-20',     26170, 22, 9,    91),
        ('21-40',    21790, 18, 30,   70),
        ('41-60',    17409, 15, 50,   50),
        ('61-80',    13402, 11, 69,   31),
        ('80+',      11021, 9,  90,   9),
    ]
    parts = [f"('{SNAP_SOLD}', '{b}', {num(t)}, {num(p)}, {num(s)}, {num(u)})"
             for (b, t, p, s, u) in so_prog]
    psql(f"""
        INSERT INTO domrf_sold_out_by_progress (snapshot_date, progress_bucket, total_th_sqm,
            pct_of_total, sold_pct_within, unsold_pct_within)
        VALUES {','.join(parts)}
        ON CONFLICT (snapshot_date, progress_bucket) DO UPDATE SET
            total_th_sqm = EXCLUDED.total_th_sqm,
            pct_of_total = EXCLUDED.pct_of_total,
            sold_pct_within = EXCLUDED.sold_pct_within,
            unsold_pct_within = EXCLUDED.unsold_pct_within;
    """)
    print(f'sold_out_by_progress: {len(parts)}')

    # ─────────── 6. mortgage_rates.pdf → domrf_mortgage_rates ───────────
    rates = [
        # (bank, primary, secondary, refi, note)
        ('Сбербанк',                       20.2,   19.7,  21.5,  None),
        ('ВТБ',                            19.9,   19.9,  19.6,  None),
        ('Банк ДОМ.РФ',                    18.2,   18.2,  22.1,  None),
        ('Альфа-банк',                     19.49,  19.49, 18.99, None),
        ('Совкомбанк',                     19.99,  20.49, 19.99, None),
        ('Т-Банк',                         16.9,   16.9,  16.9,  None),
        ('Банк Санкт-Петербург',           18.49,  18.49, 18.49, None),
        ('Промсвязьбанк',                  19.49,  18.79, 18.99, None),
        ('Московский Кредитный Банк',      18.6,   18.6,  18.6,  None),
        ('Уралсиб',                        17.89,  17.89, 18.99, None),
        ('Абсолют банк',                   19.35,  19.35, 19.35, None),
        ('Металлинвестбанк',               18.1,   18.1,  18.5,  'к пред.неделе -0.7 пп'),
        ('Газпромбанк',                    None,   None,  None,  'Максимальный срок – 19 лет, рефинанс не предоставляет'),
        ('Азиатско-Тихоокеанский Банк',    18.9,   18.9,  18.9,  None),
        ('Россельхозбанк',                 None,   None,  None,  'не предоставляет'),
        ('Банк Кубань Кредит',             16.4,   16.4,  15.9,  None),
        ('Транскапиталбанк',               19.6,   19.6,  19.6,  None),
        ('УБРиР',                          None,   None,  None,  'не предоставляет'),
        ('Ак Барс Банк',                   None,   None,  None,  'ПВ от 50%, рефинанс не предоставляет'),
        ('Средневзвешенная',               19.90,  19.59, 18.81, 'aggregate top-20'),
    ]
    parts = [f"('{SNAP_MORTGAGE}', {esc(b)}, {num(p)}, {num(s)}, {num(r)}, {esc(n)})"
             for (b, p, s, r, n) in rates]
    psql(f"""
        INSERT INTO domrf_mortgage_rates (snapshot_date, bank_name, primary_rate, secondary_rate, refinance_rate, note)
        VALUES {','.join(parts)}
        ON CONFLICT (snapshot_date, bank_name) DO UPDATE SET
            primary_rate = EXCLUDED.primary_rate,
            secondary_rate = EXCLUDED.secondary_rate,
            refinance_rate = EXCLUDED.refinance_rate,
            note = EXCLUDED.note;
    """)
    print(f'mortgage_rates: {len(parts)}')

    # Summary
    print('\n--- summary ---')
    out = psql("""
        SELECT 'developers_full' tab, COUNT(*) FROM domrf_developers_full
        UNION ALL SELECT 'escrow_banks', COUNT(*) FROM domrf_escrow_banks
        UNION ALL SELECT 'guaranty_regions', COUNT(*) FROM domrf_guaranty_regions
        UNION ALL SELECT 'housing_summary', COUNT(*) FROM domrf_housing_summary
        UNION ALL SELECT 'planned_commissioning', COUNT(*) FROM domrf_planned_commissioning
        UNION ALL SELECT 'sold_out', COUNT(*) FROM domrf_sold_out
        UNION ALL SELECT 'sold_out_by_year', COUNT(*) FROM domrf_sold_out_by_year
        UNION ALL SELECT 'sold_out_by_progress', COUNT(*) FROM domrf_sold_out_by_progress
        UNION ALL SELECT 'mortgage_rates', COUNT(*) FROM domrf_mortgage_rates
        ORDER BY 1;
    """)
    print(out)


if __name__ == '__main__':
    main()
