"""Loads DOM.RF extra snapshots (XLSX downloads + PDF-extracted data) into Postgres. Inputs in C:\\Users\\user\\Downloads\\: - developer_rf_26-04-2026.xlsx → domrf_developers_full - escrow_rf_26-04-2026.xlsx → domrf_escrow_banks - guaranty_26-04-2026.xlsx → domrf_guaranty_regions - housing_info_26-04-2026.pdf → domrf_housing_summary, domrf_planned_commissioning (hardcoded extract) - sold_out_info_26-04-2026.pdf → domrf_sold_out, ..._by_year, ..._by_progress (hardcoded extract) - mortgage_rates_19-04-2026.pdf → domrf_mortgage_rates (hardcoded extract) Usage: PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... python data/sql/21_load_domrf_extras.py """ import os, sys, subprocess import openpyxl DOWNLOADS = r'C:\Users\user\Downloads' HERE = os.path.dirname(os.path.abspath(__file__)) PG_PORT = os.environ.get('PG_PORT', '5432') PG_USER = os.environ.get('PG_USER', 'gendesign') PG_DB = os.environ.get('PG_DB', 'gendesign') PG_PASS = os.environ.get('PGPASSWORD', '') def psql(sql): cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}', 'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT, '-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet'] res = subprocess.run(cmd, input=sql, capture_output=True, text=True, encoding='utf-8') if res.returncode != 0: print('psql FAILED:', res.stderr[-2000:], file=sys.stderr) raise SystemExit(res.returncode) return res.stdout def esc(s): if s is None: return 'NULL' return "'" + str(s).replace("'", "''") + "'" def num(v): if v is None or v == '-' or v == '': return 'NULL' try: return str(float(v)) except (ValueError, TypeError): return 'NULL' def integer(v): if v is None or v == '-' or v == '': return 'NULL' try: return str(int(float(v))) except (ValueError, TypeError): return 'NULL' def main(): # 0. Apply schema schema = open(os.path.join(HERE, '20_schema_domrf_extras.sql'), encoding='utf-8').read() psql(schema) print('schema applied') SNAP_APR26 = '2026-04-26' SNAP_MORTGAGE = '2026-04-19' SNAP_SOLD = '2026-04-11' # отчётная дата на самом PDF # ─────────── 1. developer_rf.xlsx → domrf_developers_full ─────────── wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'developer_rf_26-04-2026.xlsx'), read_only=True, data_only=True) rows = list(wb.active.iter_rows(values_only=True)) wb.close() # rows[3] = header; rows[4] = numeric column index; rows[5] = 'Итого'; rows[6+] = data seen = {} for r in rows[6:]: name = r[0] if not name or not isinstance(name, str) or name.strip() == '': continue # Deduplicate: keep entry with highest area (xlsx может содержать дубли названий) key = name.strip() prev = seen.get(key) cur_area = r[1] if r[1] is not None else 0 if prev is None or (prev[1] or 0) < cur_area: seen[key] = r parts = [f"('{SNAP_APR26}', {esc(k)}, {num(r[1])}, {integer(r[2])}, " f"{integer(r[3])}, {integer(r[4])}, {num(r[5])})" for k, r in seen.items()] BATCH = 500 for i in range(0, len(parts), BATCH): psql(f""" INSERT INTO domrf_developers_full (snapshot_date, developer_name, area_thousand_sqm, permits_count, houses_count, flats_count, market_share_pct) VALUES {','.join(parts[i:i+BATCH])} ON CONFLICT (snapshot_date, developer_name) DO UPDATE SET area_thousand_sqm = EXCLUDED.area_thousand_sqm, permits_count = EXCLUDED.permits_count, houses_count = EXCLUDED.houses_count, flats_count = EXCLUDED.flats_count, market_share_pct = EXCLUDED.market_share_pct; """) print(f'developers_full: {len(parts)} (deduped from {len(rows)-6})') # ─────────── 2. escrow_rf.xlsx → domrf_escrow_banks ─────────── wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'escrow_rf_26-04-2026.xlsx'), read_only=True, data_only=True) rows = list(wb.active.iter_rows(values_only=True)) wb.close() parts = [] for r in rows[6:]: name = r[0] if not name or not isinstance(name, str) or name.strip() == '' or name == 'Итого': continue parts.append(f"('{SNAP_APR26}', {esc(name.strip())}, {integer(r[1])}, {integer(r[2])}, " f"{num(r[3])}, {integer(r[4])}, {integer(r[5])})") psql(f""" INSERT INTO domrf_escrow_banks (snapshot_date, bank_name, houses_count, permits_count, area_thousand_sqm, developers_count, regions_count) VALUES {','.join(parts)} ON CONFLICT (snapshot_date, bank_name) DO UPDATE SET houses_count = EXCLUDED.houses_count, permits_count = EXCLUDED.permits_count, area_thousand_sqm = EXCLUDED.area_thousand_sqm, developers_count = EXCLUDED.developers_count, regions_count = EXCLUDED.regions_count; """) print(f'escrow_banks: {len(parts)}') # ─────────── 3. guaranty.xlsx → domrf_guaranty_regions ─────────── wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'guaranty_26-04-2026.xlsx'), read_only=True, data_only=True) rows = list(wb.active.iter_rows(values_only=True)) wb.close() # rows[6] = РФ; rows[7..] = ФО + регионы FO_NAMES = {'Центральный ФО', 'Северо-Западный ФО', 'Южный ФО', 'Северо-Кавказский ФО', 'Приволжский ФО', 'Уральский ФО', 'Сибирский ФО', 'Дальневосточный ФО', 'Новые субъекты Российской Федерации'} parts = [] for r in rows[6:]: name = r[0] if not name or not isinstance(name, str) or name.strip() == '': continue name = name.strip() if name == 'Российская Федерация': level = 'rf' elif name in FO_NAMES: level = 'fo' else: level = 'region' parts.append(f"('{SNAP_APR26}', {esc(name)}, '{level}', " f"{integer(r[1])}, {num(r[2])}, {integer(r[3])}, " f"{integer(r[4])}, {num(r[5])}, {integer(r[6])}, {num(r[7])}, " f"{integer(r[8])}, {num(r[9])}, {integer(r[10])}, " f"{integer(r[11])}, {num(r[12])}, {integer(r[13])})") BATCH = 100 for i in range(0, len(parts), BATCH): psql(f""" INSERT INTO domrf_guaranty_regions (snapshot_date, territory_name, territory_level, total_permits, total_area_th_sqm, total_developers, fz214_permits, fz214_area_th_sqm, fz214_developers, fz214_pct_of_total, escrow_permits, escrow_area_th_sqm, escrow_developers, pp480_permits, pp480_area_th_sqm, pp480_developers) VALUES {','.join(parts[i:i+BATCH])} ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET total_permits = EXCLUDED.total_permits, total_area_th_sqm = EXCLUDED.total_area_th_sqm, total_developers = EXCLUDED.total_developers, fz214_permits = EXCLUDED.fz214_permits, fz214_area_th_sqm = EXCLUDED.fz214_area_th_sqm, fz214_developers = EXCLUDED.fz214_developers, fz214_pct_of_total = EXCLUDED.fz214_pct_of_total, escrow_permits = EXCLUDED.escrow_permits, escrow_area_th_sqm = EXCLUDED.escrow_area_th_sqm, escrow_developers = EXCLUDED.escrow_developers, pp480_permits = EXCLUDED.pp480_permits, pp480_area_th_sqm = EXCLUDED.pp480_area_th_sqm, pp480_developers = EXCLUDED.pp480_developers; """) print(f'guaranty_regions: {len(parts)}') # ─────────── 4. housing_info.pdf → domrf_housing_summary + planned_commissioning ─────────── # extracted manually from PDF housing_summary = [ # (territory, level, mechanism, devs, permits, decls, houses, area, flats_th) ('Российская Федерация', 'rf', 'all', 4336, 7257, 7447, 11510, 119234, 2424), ('Российская Федерация', 'rf', 'escrow', 4285, 7170, 7359, 11385, 117670, 2394), ('Российская Федерация', 'rf', 'fund', 42, 61, 62, 85, 1104, 22), ('Российская Федерация', 'rf', 'no_attraction', 15, 28, 28, 40, 460, 8), ] # top-10 regions (escrow / fund / no_attraction / total) area_th_sqm region_breakdown = [ # (region, escrow, fund, no_attr, total) ('Город Москва', 15670, 364, 286, 16320), ('Краснодарский край', 8179, 24, 4, 8206), ('Московская область', 7905, 81, None, 7986), ('Свердловская область', 5637, 17, None, 5654), ('Город Санкт-Петербург', 5258, None, None, 5258), ('Ростовская область', 4926, None, None, 4926), ('Ленинградская область', 4640, 60, None, 4700), ('Новосибирская область', 3580, 107, None, 3687), ('Республика Башкортостан',3583, None, None, 3583), ('Тюменская область', 3513, None, 11, 3524), ] city_breakdown = [ # (city, escrow, fund, no_attr, total) ('Москва', 15670, 364, 286, 16320), ('Краснодар', 5791, None, None, 5791), ('Екатеринбург', 5294, 17, None, 5310), ('Санкт-Петербург', 5258, None, None, 5258), ('Ростов-на-Дону', 3605, None, None, 3605), ('Тюмень', 3204, None, 11, 3215), ('Новосибирск', 2755, 104, None, 2859), ('Уфа', 2786, None, None, 2786), ('Казань', 2205, None, None, 2205), ('Владивосток', 2134, None, None, 2134), ] parts = [] for (terr, level, mech, devs, permits, decls, houses, area, flats) in housing_summary: parts.append(f"('{SNAP_APR26}', {esc(terr)}, '{level}', '{mech}', {integer(devs)}, " f"{integer(permits)}, {integer(decls)}, {integer(houses)}, {num(area)}, {num(flats)})") for breakdown, level in [(region_breakdown, 'region'), (city_breakdown, 'city')]: for (terr, escrow, fund, no_attr, total) in breakdown: for mech, area in [('escrow', escrow), ('fund', fund), ('no_attraction', no_attr), ('all', total)]: if area is None: continue parts.append(f"('{SNAP_APR26}', {esc(terr)}, '{level}', '{mech}', NULL, " f"NULL, NULL, NULL, {num(area)}, NULL)") psql(f""" INSERT INTO domrf_housing_summary (snapshot_date, territory_name, territory_level, mechanism, developers_count, permits_count, declarations_count, houses_count, area_thousand_sqm, flats_thousand) VALUES {','.join(parts)} ON CONFLICT (snapshot_date, territory_name, mechanism) DO UPDATE SET developers_count = EXCLUDED.developers_count, permits_count = EXCLUDED.permits_count, declarations_count = EXCLUDED.declarations_count, houses_count = EXCLUDED.houses_count, area_thousand_sqm = EXCLUDED.area_thousand_sqm, flats_thousand = EXCLUDED.flats_thousand; """) print(f'housing_summary: {len(parts)}') # planned_commissioning (year, escrow, fund, no_attr, total) pc = [ (2026, 37084, 685, 154, 37923), (2027, 34896, 246, 221, 35363), (2028, 26360, 113, 30, 26503), (2029, 10924, 19, 24, 10967), (2030, 4245, 13, 32, 4290), (2031, 1797, 28, None, 1825), (2032, 1323, None, None, 1323), (2033, 351, None, None, 351), (2034, 234, None, None, 234), (2035, 67, None, None, 67), ] parts = [f"('{SNAP_APR26}', {y}, {num(e)}, {num(f)}, {num(n)}, {num(t)})" for (y, e, f, n, t) in pc] psql(f""" INSERT INTO domrf_planned_commissioning (snapshot_date, plan_year, escrow_th_sqm, fund_th_sqm, no_attraction_th_sqm, total_th_sqm) VALUES {','.join(parts)} ON CONFLICT (snapshot_date, plan_year) DO UPDATE SET escrow_th_sqm = EXCLUDED.escrow_th_sqm, fund_th_sqm = EXCLUDED.fund_th_sqm, no_attraction_th_sqm = EXCLUDED.no_attraction_th_sqm, total_th_sqm = EXCLUDED.total_th_sqm; """) print(f'planned_commissioning: {len(parts)}') # ─────────── 5. sold_out_info.pdf → 3 tables ─────────── # RF total psql(f""" INSERT INTO domrf_sold_out (snapshot_date, territory_name, total_area_th_sqm, sold_area_th_sqm, unsold_area_th_sqm, not_open_area_th_sqm, sold_pct, unsold_pct, not_open_pct, avg_price_per_sqm, attracted_funds_mln_rub) VALUES ('{SNAP_SOLD}', 'Российская Федерация', 118876, 36749, 53044, 29084, 31, 45, 24, 216316, 7949316) ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET total_area_th_sqm = EXCLUDED.total_area_th_sqm, sold_area_th_sqm = EXCLUDED.sold_area_th_sqm, unsold_area_th_sqm = EXCLUDED.unsold_area_th_sqm, not_open_area_th_sqm = EXCLUDED.not_open_area_th_sqm, sold_pct = EXCLUDED.sold_pct, unsold_pct = EXCLUDED.unsold_pct, not_open_pct = EXCLUDED.not_open_pct, avg_price_per_sqm = EXCLUDED.avg_price_per_sqm, attracted_funds_mln_rub = EXCLUDED.attracted_funds_mln_rub; """) # Top-8 regions: (region, total, sold%, unsold%, not_open%) so_regions = [ ('Город Москва', 16405, 47, 44, None), ('Краснодарский край', 8286, 20, 39, 41), ('Московская область', 7942, 37, 43, None), ('Свердловская область', 5686, 29, 54, None), ('Город Санкт-Петербург', 5336, 40, 44, None), ('Ростовская область', 4990, None, 48, None), ('Ленинградская область', 4705, None, 44, None), ('Тюменская область', 3573, None, 50, None), ] parts = [] for (r, total, sp, up, nop) in so_regions: parts.append(f"('{SNAP_SOLD}', {esc(r)}, {num(total)}, NULL, NULL, NULL, " f"{num(sp)}, {num(up)}, {num(nop)}, NULL, NULL)") psql(f""" INSERT INTO domrf_sold_out (snapshot_date, territory_name, total_area_th_sqm, sold_area_th_sqm, unsold_area_th_sqm, not_open_area_th_sqm, sold_pct, unsold_pct, not_open_pct, avg_price_per_sqm, attracted_funds_mln_rub) VALUES {','.join(parts)} ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET total_area_th_sqm = EXCLUDED.total_area_th_sqm, sold_pct = EXCLUDED.sold_pct, unsold_pct = EXCLUDED.unsold_pct, not_open_pct = EXCLUDED.not_open_pct; """) print(f'sold_out: 1 RF + {len(parts)} regions') # by_year — (year, total_th_sqm, pct, sold%, unsold%, not_open%) so_year = [ ('2026', 40201, 34, 52, 38, 10), ('2027', 34880, 29, 30, 53, 17), ('2028', 25363, 21, 17, 52, 31), ('2029', 10359, 9, None, 37, 54), ('2030', 4127, 3, None, None, 71), ('2031+', 3946, 3, None, None, 65), ] parts = [f"('{SNAP_SOLD}', '{y}', {num(t)}, {num(p)}, {num(s)}, {num(u)}, {num(n)})" for (y, t, p, s, u, n) in so_year] psql(f""" INSERT INTO domrf_sold_out_by_year (snapshot_date, plan_year, total_th_sqm, pct_of_total, sold_pct, unsold_pct, not_open_pct) VALUES {','.join(parts)} ON CONFLICT (snapshot_date, plan_year) DO UPDATE SET total_th_sqm = EXCLUDED.total_th_sqm, pct_of_total = EXCLUDED.pct_of_total, sold_pct = EXCLUDED.sold_pct, unsold_pct = EXCLUDED.unsold_pct, not_open_pct = EXCLUDED.not_open_pct; """) print(f'sold_out_by_year: {len(parts)}') # by_progress so_prog = [ ('not_open', 29084, 24, None, 100), ('0-20', 26170, 22, 9, 91), ('21-40', 21790, 18, 30, 70), ('41-60', 17409, 15, 50, 50), ('61-80', 13402, 11, 69, 31), ('80+', 11021, 9, 90, 9), ] parts = [f"('{SNAP_SOLD}', '{b}', {num(t)}, {num(p)}, {num(s)}, {num(u)})" for (b, t, p, s, u) in so_prog] psql(f""" INSERT INTO domrf_sold_out_by_progress (snapshot_date, progress_bucket, total_th_sqm, pct_of_total, sold_pct_within, unsold_pct_within) VALUES {','.join(parts)} ON CONFLICT (snapshot_date, progress_bucket) DO UPDATE SET total_th_sqm = EXCLUDED.total_th_sqm, pct_of_total = EXCLUDED.pct_of_total, sold_pct_within = EXCLUDED.sold_pct_within, unsold_pct_within = EXCLUDED.unsold_pct_within; """) print(f'sold_out_by_progress: {len(parts)}') # ─────────── 6. mortgage_rates.pdf → domrf_mortgage_rates ─────────── rates = [ # (bank, primary, secondary, refi, note) ('Сбербанк', 20.2, 19.7, 21.5, None), ('ВТБ', 19.9, 19.9, 19.6, None), ('Банк ДОМ.РФ', 18.2, 18.2, 22.1, None), ('Альфа-банк', 19.49, 19.49, 18.99, None), ('Совкомбанк', 19.99, 20.49, 19.99, None), ('Т-Банк', 16.9, 16.9, 16.9, None), ('Банк Санкт-Петербург', 18.49, 18.49, 18.49, None), ('Промсвязьбанк', 19.49, 18.79, 18.99, None), ('Московский Кредитный Банк', 18.6, 18.6, 18.6, None), ('Уралсиб', 17.89, 17.89, 18.99, None), ('Абсолют банк', 19.35, 19.35, 19.35, None), ('Металлинвестбанк', 18.1, 18.1, 18.5, 'к пред.неделе -0.7 пп'), ('Газпромбанк', None, None, None, 'Максимальный срок – 19 лет, рефинанс не предоставляет'), ('Азиатско-Тихоокеанский Банк', 18.9, 18.9, 18.9, None), ('Россельхозбанк', None, None, None, 'не предоставляет'), ('Банк Кубань Кредит', 16.4, 16.4, 15.9, None), ('Транскапиталбанк', 19.6, 19.6, 19.6, None), ('УБРиР', None, None, None, 'не предоставляет'), ('Ак Барс Банк', None, None, None, 'ПВ от 50%, рефинанс не предоставляет'), ('Средневзвешенная', 19.90, 19.59, 18.81, 'aggregate top-20'), ] parts = [f"('{SNAP_MORTGAGE}', {esc(b)}, {num(p)}, {num(s)}, {num(r)}, {esc(n)})" for (b, p, s, r, n) in rates] psql(f""" INSERT INTO domrf_mortgage_rates (snapshot_date, bank_name, primary_rate, secondary_rate, refinance_rate, note) VALUES {','.join(parts)} ON CONFLICT (snapshot_date, bank_name) DO UPDATE SET primary_rate = EXCLUDED.primary_rate, secondary_rate = EXCLUDED.secondary_rate, refinance_rate = EXCLUDED.refinance_rate, note = EXCLUDED.note; """) print(f'mortgage_rates: {len(parts)}') # Summary print('\n--- summary ---') out = psql(""" SELECT 'developers_full' tab, COUNT(*) FROM domrf_developers_full UNION ALL SELECT 'escrow_banks', COUNT(*) FROM domrf_escrow_banks UNION ALL SELECT 'guaranty_regions', COUNT(*) FROM domrf_guaranty_regions UNION ALL SELECT 'housing_summary', COUNT(*) FROM domrf_housing_summary UNION ALL SELECT 'planned_commissioning', COUNT(*) FROM domrf_planned_commissioning UNION ALL SELECT 'sold_out', COUNT(*) FROM domrf_sold_out UNION ALL SELECT 'sold_out_by_year', COUNT(*) FROM domrf_sold_out_by_year UNION ALL SELECT 'sold_out_by_progress', COUNT(*) FROM domrf_sold_out_by_progress UNION ALL SELECT 'mortgage_rates', COUNT(*) FROM domrf_mortgage_rates ORDER BY 1; """) print(out) if __name__ == '__main__': main()