"""Migrate Anton's domrf.db (SQLite, 23.5 MB) into prod Postgres.

Reads:  C:\\Users\\user\\Downloads\\Telegram Desktop\\db\\db\\domrf.db
Writes: 13 new tables created by 43_anton_import.sql + merges developers.

Skipped tables (duplicates of our existing data):
  • regions, federal_districts (we have domrf_regions/_federal_districts)
  • mortgage_rates (empty in Anton's DB, we have domrf_mortgage_rates)
  • escrow_banks (we have domrf_escrow_banks, 28 vs 27)
  • rc_classes/dev_size/city_pop (in our domrf_sold_ready_breakdown)
  • rc_dynamics_rf (in our domrf_sold_ready_dynamics)
  • egrn_deals (we have raw rosreestr_deals — 4.97M rows, far richer)
  • _meta, _sources, sqlite_sequence, launch_5y(empty)

Special handling:
  • developers (8 618) — merge missing rows into domrf_developers (we have 2 734)
"""
import os, sys, sqlite3, subprocess, datetime

SQLITE_PATH = r"C:\Users\user\Downloads\Telegram Desktop\db\db\domrf.db"
PG_HOST = os.environ.get('PG_HOST', 'host.docker.internal')
PG_PORT = os.environ.get('PG_PORT', '15432')
PG_USER = os.environ.get('PG_USER', 'gendesign')
PG_DB   = os.environ.get('PG_DB',   'gendesign')
PG_PASS = os.environ.get('PGPASSWORD', '2J2SBPMKuS998fiwhtQqDhMI')


def psql(sql):
    cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
           'postgres:16-alpine', 'psql',
           '-h', PG_HOST, '-p', PG_PORT, '-U', PG_USER, '-d', PG_DB,
           '-v', 'ON_ERROR_STOP=1', '--quiet']
    res = subprocess.run(cmd, input=sql, capture_output=True, text=True, encoding='utf-8')
    if res.returncode != 0:
        print('psql FAILED:', (res.stderr or '')[-2000:], file=sys.stderr)
        raise SystemExit(res.returncode)
    return res.stdout


def esc_val(v):
    if v is None:
        return 'NULL'
    if isinstance(v, bool):
        return 'TRUE' if v else 'FALSE'
    if isinstance(v, (int, float)):
        if isinstance(v, float) and v != v:  # NaN
            return 'NULL'
        return str(v)
    return "'" + str(v).replace("'", "''") + "'"


def to_pg_bool(v):
    if v is None:
        return 'NULL'
    return 'TRUE' if v else 'FALSE'


def to_pg_date(v):
    """SQLite stores dates as TEXT. Try parse YYYY-MM-DD; else NULL."""
    if v is None or v == '':
        return 'NULL'
    s = str(v)[:10]
    try:
        datetime.date.fromisoformat(s)
        return f"'{s}'"
    except Exception:
        return 'NULL'


def chunked_insert(rows, table, cols, conflict_cols=None, batch=500, do_update=False):
    if not rows:
        return 0
    total = 0
    on_conflict = ''
    if conflict_cols:
        if do_update:
            update = ', '.join(f'{c}=EXCLUDED.{c}' for c in cols if c not in conflict_cols)
            on_conflict = f' ON CONFLICT ({",".join(conflict_cols)}) DO UPDATE SET {update}'
        else:
            on_conflict = f' ON CONFLICT ({",".join(conflict_cols)}) DO NOTHING'
    for i in range(0, len(rows), batch):
        chunk = rows[i:i + batch]
        sql = (f"INSERT INTO {table} ({','.join(cols)}) VALUES "
               + ','.join(chunk) + on_conflict + ';')
        psql(sql)
        total += len(chunk)
    return total


def migrate_kn_flats(con):
    rows = []
    for r in con.execute('SELECT id, ods_id, type, number, is_studio, total_area, '
                         'living_area, rooms, status, price_rub, price_per_m2, '
                         'floor, num_floors, obj_id, city, region_cd, obj_name '
                         'FROM flats'):
        (fid, ods, ftype, fnum, studio, ta, la, rooms, status, pr, ppm,
         fl, nfl, obj_id, city, region, obj_name) = r
        rows.append(
            f"({fid},{esc_val(ods)},{esc_val(ftype)},{esc_val(fnum)},"
            f"{to_pg_bool(studio)},{esc_val(ta)},{esc_val(la)},{esc_val(rooms)},"
            f"{esc_val(status)},{esc_val(pr)},{esc_val(ppm)},{esc_val(fl)},"
            f"{esc_val(nfl)},{esc_val(obj_id)},{esc_val(city)},{esc_val(region)},"
            f"{esc_val(obj_name)},DEFAULT)"
        )
    return chunked_insert(
        rows, 'domrf_kn_flats',
        ['id','ods_id','flat_type','flat_number','is_studio','total_area','living_area',
         'rooms','status','price_rub','price_per_m2','floor','num_floors','obj_id',
         'city','region_cd','obj_name','snapshot_date'],
        conflict_cols=['id'], do_update=True,
    )


def migrate_kn_objects(con):
    rows = []
    for r in con.execute('SELECT obj_id, hobj_id, comm_name, addr, short_addr, region_cd, '
                         'dev_id, dev_name, dev_inn, floor_min, floor_max, flat_count, '
                         'square_living, ready_dt, problem_flag, site_status, green_house, '
                         'escrow, obj_class, wall_type, energy_eff, latitude, longitude, '
                         'obj_status, is_prinzip, is_ekb, is_problem FROM objects'):
        (oid, hid, cn, addr, sa, rc, did, dn, dinn, fmn, fmx, fc, sl, rdt,
         pf, ss, gh, esc, oc, wt, ee, lat, lon, ostat, ipr, iekb, iprob) = r
        rows.append(
            f"({oid},{esc_val(hid)},{esc_val(cn)},{esc_val(addr)},{esc_val(sa)},"
            f"{esc_val(rc)},{esc_val(did)},{esc_val(dn)},{esc_val(dinn)},"
            f"{esc_val(fmn)},{esc_val(fmx)},{esc_val(fc)},{esc_val(sl)},"
            f"{to_pg_date(rdt)},{esc_val(pf)},{esc_val(ss)},"
            f"{to_pg_bool(gh)},{to_pg_bool(esc)},{esc_val(oc)},{esc_val(wt)},"
            f"{esc_val(ee)},{esc_val(lat)},{esc_val(lon)},{esc_val(ostat)},"
            f"{to_pg_bool(ipr)},{to_pg_bool(iekb)},{to_pg_bool(iprob)},DEFAULT)"
        )
    return chunked_insert(
        rows, 'domrf_kn_objects',
        ['obj_id','hobj_id','comm_name','addr','short_addr','region_cd','dev_id','dev_name',
         'dev_inn','floor_min','floor_max','flat_count','square_living','ready_dt',
         'problem_flag','site_status','green_house','escrow','obj_class','wall_type',
         'energy_eff','latitude','longitude','obj_status','is_prinzip','is_ekb','is_problem',
         'snapshot_date'],
        conflict_cols=['obj_id'], do_update=True,
    )


def migrate_simple(con, src, dst, cols_src, cols_dst, conflict_cols, do_update=True,
                   transforms=None):
    """Generic migration: SELECT cols from src → INSERT into dst."""
    transforms = transforms or {}
    rows = []
    for r in con.execute(f'SELECT {",".join(cols_src)} FROM "{src}"'):
        vals = []
        for col_name, raw in zip(cols_src, r):
            if col_name in transforms:
                vals.append(transforms[col_name](raw))
            else:
                vals.append(esc_val(raw))
        rows.append('(' + ','.join(vals) + ')')
    return chunked_insert(rows, dst, cols_dst, conflict_cols=conflict_cols, do_update=do_update)


def merge_developers(con):
    """Anton has 8 618 developers (incl. archived); we have 2 734.
    INSERT only IDs that don't exist in our table — no update on existing."""
    # Pull existing developer_ids from prod via psql
    existing_ids = set()
    res = psql('SELECT developer_id FROM domrf_developers;')
    for line in res.splitlines():
        line = line.strip()
        if line and not line.startswith('---') and 'developer_id' not in line and 'rows' not in line and '(' not in line:
            existing_ids.add(line)
    print(f'  existing developers in prod: {len(existing_ids)}')

    new_rows = []
    skipped = 0
    cur = con.execute('SELECT id, name FROM developers WHERE id IS NOT NULL')
    for did, name in cur:
        if not did or not name:
            continue
        if str(did) in existing_ids:
            skipped += 1
            continue
        new_rows.append(f"({esc_val(did)},{esc_val(name)})")
    print(f'  skipped (already in prod): {skipped}')
    print(f'  to INSERT (new from Anton): {len(new_rows)}')

    return chunked_insert(
        new_rows, 'domrf_developers', ['developer_id', 'developer_name'],
        conflict_cols=['developer_id'], do_update=False,
    )


def main():
    con = sqlite3.connect(SQLITE_PATH)

    print('--- domrf_kn_flats ---')
    n = migrate_kn_flats(con)
    print(f'  loaded {n} flats')

    print('--- domrf_kn_objects ---')
    n = migrate_kn_objects(con)
    print(f'  loaded {n} objects')

    print('--- cbr_mortgage_files ---')
    n = migrate_simple(con, 'cbr_files', 'cbr_mortgage_files',
        ['filename', 'size_kb', 'description', 'url'],
        ['filename', 'size_kb', 'description', 'url'],
        conflict_cols=['filename'])
    print(f'  loaded {n}')

    print('--- cbr_mortgage_series ---')
    n = migrate_simple(con, 'cbr_series', 'cbr_mortgage_series',
        ['series_id', 'region', 'period', 'value', 'title'],
        ['series_id', 'region', 'period', 'value', 'title'],
        conflict_cols=['series_id', 'region', 'period'])
    print(f'  loaded {n}')

    print('--- sber_ddf_metrics ---')
    n = migrate_simple(con, 'ddf_metrics', 'sber_ddf_metrics',
        ['metric_key', 'region', 'value_2019', 'value_2025', 'unit', 'source'],
        ['metric_key', 'region', 'value_2019', 'value_2025', 'unit', 'source'],
        conflict_cols=['metric_key'])
    print(f'  loaded {n}')

    print('--- yandex_realty_zk ---')
    n = migrate_simple(con, 'yandex_zk', 'yandex_realty_zk',
        ['yid', 'name', 'developer', 'developer_id', 'finished_obj', 'unfinished_obj',
         'price_from', 'price_to', 'address', 'obj_class', 'latitude', 'longitude'],
        ['yid', 'name', 'developer', 'developer_id', 'finished_obj', 'unfinished_obj',
         'price_from', 'price_to', 'address', 'obj_class', 'latitude', 'longitude'],
        conflict_cols=['yid'])
    print(f'  loaded {n}')

    print('--- yandex_realty_class_prices ---')
    n = migrate_simple(con, 'yandex_class_prices', 'yandex_realty_class_prices',
        ['obj_class', 'total_sites', 'price_per_m2_min', 'price_per_m2_max'],
        ['obj_class', 'total_sites', 'price_per_m2_min', 'price_per_m2_max'],
        conflict_cols=['obj_class'])
    print(f'  loaded {n}')

    print('--- ekb_districts ---')
    n = migrate_simple(con, 'ekb_districts', 'ekb_districts',
        ['name', 'population', 'zk_count', 'flat_count', 'area_m2',
         'median_price_per_m2', 'mean_price_per_m2'],
        ['district_name', 'population', 'zk_count', 'flat_count', 'area_m2',
         'median_price_per_m2', 'mean_price_per_m2'],
        conflict_cols=['district_name'])
    print(f'  loaded {n}')

    print('--- ekb_metro ---')
    n = migrate_simple(con, 'ekb_metro', 'ekb_metro',
        ['station_name', 'zk_count', 'median_price_per_m2'],
        ['station_name', 'zk_count', 'median_price_per_m2'],
        conflict_cols=['station_name'])
    print(f'  loaded {n}')

    print('--- sv_cities ---')
    n = migrate_simple(con, 'sv_cities', 'sv_cities',
        ['city_name', 'zk_count', 'flat_count', 'area_m2', 'median_price_per_m2'],
        ['city_name', 'zk_count', 'flat_count', 'area_m2', 'median_price_per_m2'],
        conflict_cols=['city_name'])
    print(f'  loaded {n}')

    print('--- sv_dev_sales ---')
    n = migrate_simple(con, 'sv_dev_sales', 'sv_dev_sales',
        ['rank', 'dev_id', 'name', 'total_square', 'sold_perc', 'unsold_perc',
         'unopened_perc', 'rep_period'],
        ['rank', 'dev_id', 'name', 'total_square', 'sold_perc', 'unsold_perc',
         'unopened_perc', 'rep_period'],
        conflict_cols=['dev_id', 'rep_period'])
    print(f'  loaded {n}')

    print('--- domrf_dev_growth ---')
    n = migrate_simple(con, 'dev_growth', 'domrf_dev_growth',
        ['dev_id', 'dev_name', 'area_2024', 'area_2025', 'delta_pct', 'region'],
        ['dev_id', 'dev_name', 'area_2024', 'area_2025', 'delta_pct', 'region'],
        conflict_cols=['dev_name', 'region'])
    print(f'  loaded {n}')

    print('--- rosreestr_rent_deals ---')
    rows = []
    for r in con.execute('SELECT quarter, obj_type, cad_quarter, area, rent_rub, '
                         'rent_per_m2, duration_months, region_cd FROM rent_deals'):
        rows.append('(' + ','.join(esc_val(v) for v in r) + ')')
    n = chunked_insert(
        rows, 'rosreestr_rent_deals',
        ['quarter','obj_type','cad_quarter','area','rent_rub','rent_per_m2',
         'duration_months','region_cd'],
        conflict_cols=None,  # no PK conflict — id is BIGSERIAL
    )
    print(f'  loaded {n}')

    print('--- merge developers ---')
    n = merge_developers(con)
    print(f'  inserted {n} new developers')

    con.close()


if __name__ == '__main__':
    main()
