gendesign/data/sql/21_load_domrf_extras.py
2026-04-26 22:30:52 +03:00

405 lines
21 KiB
Python
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Loads DOM.RF extra snapshots (XLSX downloads + PDF-extracted data) into Postgres.
Inputs in C:\\Users\\user\\Downloads\\:
- developer_rf_26-04-2026.xlsx → domrf_developers_full
- escrow_rf_26-04-2026.xlsx → domrf_escrow_banks
- guaranty_26-04-2026.xlsx → domrf_guaranty_regions
- housing_info_26-04-2026.pdf → domrf_housing_summary, domrf_planned_commissioning (hardcoded extract)
- sold_out_info_26-04-2026.pdf → domrf_sold_out, ..._by_year, ..._by_progress (hardcoded extract)
- mortgage_rates_19-04-2026.pdf → domrf_mortgage_rates (hardcoded extract)
Usage:
PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... python data/sql/21_load_domrf_extras.py
"""
import os, sys, subprocess
import openpyxl
DOWNLOADS = r'C:\Users\user\Downloads'
HERE = os.path.dirname(os.path.abspath(__file__))
PG_PORT = os.environ.get('PG_PORT', '5432')
PG_USER = os.environ.get('PG_USER', 'gendesign')
PG_DB = os.environ.get('PG_DB', 'gendesign')
PG_PASS = os.environ.get('PGPASSWORD', '')
def psql(sql):
cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT,
'-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet']
res = subprocess.run(cmd, input=sql, capture_output=True, text=True, encoding='utf-8')
if res.returncode != 0:
print('psql FAILED:', res.stderr[-2000:], file=sys.stderr)
raise SystemExit(res.returncode)
return res.stdout
def esc(s):
if s is None: return 'NULL'
return "'" + str(s).replace("'", "''") + "'"
def num(v):
if v is None or v == '-' or v == '': return 'NULL'
try: return str(float(v))
except (ValueError, TypeError): return 'NULL'
def integer(v):
if v is None or v == '-' or v == '': return 'NULL'
try: return str(int(float(v)))
except (ValueError, TypeError): return 'NULL'
def main():
# 0. Apply schema
schema = open(os.path.join(HERE, '20_schema_domrf_extras.sql'), encoding='utf-8').read()
psql(schema)
print('schema applied')
SNAP_APR26 = '2026-04-26'
SNAP_MORTGAGE = '2026-04-19'
SNAP_SOLD = '2026-04-11' # отчётная дата на самом PDF
# ─────────── 1. developer_rf.xlsx → domrf_developers_full ───────────
wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'developer_rf_26-04-2026.xlsx'),
read_only=True, data_only=True)
rows = list(wb.active.iter_rows(values_only=True))
wb.close()
# rows[3] = header; rows[4] = numeric column index; rows[5] = 'Итого'; rows[6+] = data
seen = {}
for r in rows[6:]:
name = r[0]
if not name or not isinstance(name, str) or name.strip() == '': continue
# Deduplicate: keep entry with highest area (xlsx может содержать дубли названий)
key = name.strip()
prev = seen.get(key)
cur_area = r[1] if r[1] is not None else 0
if prev is None or (prev[1] or 0) < cur_area:
seen[key] = r
parts = [f"('{SNAP_APR26}', {esc(k)}, {num(r[1])}, {integer(r[2])}, "
f"{integer(r[3])}, {integer(r[4])}, {num(r[5])})"
for k, r in seen.items()]
BATCH = 500
for i in range(0, len(parts), BATCH):
psql(f"""
INSERT INTO domrf_developers_full (snapshot_date, developer_name, area_thousand_sqm,
permits_count, houses_count, flats_count, market_share_pct)
VALUES {','.join(parts[i:i+BATCH])}
ON CONFLICT (snapshot_date, developer_name) DO UPDATE SET
area_thousand_sqm = EXCLUDED.area_thousand_sqm,
permits_count = EXCLUDED.permits_count,
houses_count = EXCLUDED.houses_count,
flats_count = EXCLUDED.flats_count,
market_share_pct = EXCLUDED.market_share_pct;
""")
print(f'developers_full: {len(parts)} (deduped from {len(rows)-6})')
# ─────────── 2. escrow_rf.xlsx → domrf_escrow_banks ───────────
wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'escrow_rf_26-04-2026.xlsx'),
read_only=True, data_only=True)
rows = list(wb.active.iter_rows(values_only=True))
wb.close()
parts = []
for r in rows[6:]:
name = r[0]
if not name or not isinstance(name, str) or name.strip() == '' or name == 'Итого': continue
parts.append(f"('{SNAP_APR26}', {esc(name.strip())}, {integer(r[1])}, {integer(r[2])}, "
f"{num(r[3])}, {integer(r[4])}, {integer(r[5])})")
psql(f"""
INSERT INTO domrf_escrow_banks (snapshot_date, bank_name, houses_count, permits_count,
area_thousand_sqm, developers_count, regions_count)
VALUES {','.join(parts)}
ON CONFLICT (snapshot_date, bank_name) DO UPDATE SET
houses_count = EXCLUDED.houses_count,
permits_count = EXCLUDED.permits_count,
area_thousand_sqm = EXCLUDED.area_thousand_sqm,
developers_count = EXCLUDED.developers_count,
regions_count = EXCLUDED.regions_count;
""")
print(f'escrow_banks: {len(parts)}')
# ─────────── 3. guaranty.xlsx → domrf_guaranty_regions ───────────
wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'guaranty_26-04-2026.xlsx'),
read_only=True, data_only=True)
rows = list(wb.active.iter_rows(values_only=True))
wb.close()
# rows[6] = РФ; rows[7..] = ФО + регионы
FO_NAMES = {'Центральный ФО', 'Северо-Западный ФО', 'Южный ФО', 'Северо-Кавказский ФО',
'Приволжский ФО', 'Уральский ФО', 'Сибирский ФО', 'Дальневосточный ФО',
'Новые субъекты Российской Федерации'}
parts = []
for r in rows[6:]:
name = r[0]
if not name or not isinstance(name, str) or name.strip() == '': continue
name = name.strip()
if name == 'Российская Федерация': level = 'rf'
elif name in FO_NAMES: level = 'fo'
else: level = 'region'
parts.append(f"('{SNAP_APR26}', {esc(name)}, '{level}', "
f"{integer(r[1])}, {num(r[2])}, {integer(r[3])}, "
f"{integer(r[4])}, {num(r[5])}, {integer(r[6])}, {num(r[7])}, "
f"{integer(r[8])}, {num(r[9])}, {integer(r[10])}, "
f"{integer(r[11])}, {num(r[12])}, {integer(r[13])})")
BATCH = 100
for i in range(0, len(parts), BATCH):
psql(f"""
INSERT INTO domrf_guaranty_regions (snapshot_date, territory_name, territory_level,
total_permits, total_area_th_sqm, total_developers,
fz214_permits, fz214_area_th_sqm, fz214_developers, fz214_pct_of_total,
escrow_permits, escrow_area_th_sqm, escrow_developers,
pp480_permits, pp480_area_th_sqm, pp480_developers)
VALUES {','.join(parts[i:i+BATCH])}
ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
total_permits = EXCLUDED.total_permits,
total_area_th_sqm = EXCLUDED.total_area_th_sqm,
total_developers = EXCLUDED.total_developers,
fz214_permits = EXCLUDED.fz214_permits,
fz214_area_th_sqm = EXCLUDED.fz214_area_th_sqm,
fz214_developers = EXCLUDED.fz214_developers,
fz214_pct_of_total = EXCLUDED.fz214_pct_of_total,
escrow_permits = EXCLUDED.escrow_permits,
escrow_area_th_sqm = EXCLUDED.escrow_area_th_sqm,
escrow_developers = EXCLUDED.escrow_developers,
pp480_permits = EXCLUDED.pp480_permits,
pp480_area_th_sqm = EXCLUDED.pp480_area_th_sqm,
pp480_developers = EXCLUDED.pp480_developers;
""")
print(f'guaranty_regions: {len(parts)}')
# ─────────── 4. housing_info.pdf → domrf_housing_summary + planned_commissioning ───────────
# extracted manually from PDF
housing_summary = [
# (territory, level, mechanism, devs, permits, decls, houses, area, flats_th)
('Российская Федерация', 'rf', 'all', 4336, 7257, 7447, 11510, 119234, 2424),
('Российская Федерация', 'rf', 'escrow', 4285, 7170, 7359, 11385, 117670, 2394),
('Российская Федерация', 'rf', 'fund', 42, 61, 62, 85, 1104, 22),
('Российская Федерация', 'rf', 'no_attraction', 15, 28, 28, 40, 460, 8),
]
# top-10 regions (escrow / fund / no_attraction / total) area_th_sqm
region_breakdown = [
# (region, escrow, fund, no_attr, total)
('Город Москва', 15670, 364, 286, 16320),
('Краснодарский край', 8179, 24, 4, 8206),
('Московская область', 7905, 81, None, 7986),
('Свердловская область', 5637, 17, None, 5654),
('Город Санкт-Петербург', 5258, None, None, 5258),
('Ростовская область', 4926, None, None, 4926),
('Ленинградская область', 4640, 60, None, 4700),
('Новосибирская область', 3580, 107, None, 3687),
('Республика Башкортостан',3583, None, None, 3583),
('Тюменская область', 3513, None, 11, 3524),
]
city_breakdown = [
# (city, escrow, fund, no_attr, total)
('Москва', 15670, 364, 286, 16320),
('Краснодар', 5791, None, None, 5791),
('Екатеринбург', 5294, 17, None, 5310),
('Санкт-Петербург', 5258, None, None, 5258),
('Ростов-на-Дону', 3605, None, None, 3605),
('Тюмень', 3204, None, 11, 3215),
('Новосибирск', 2755, 104, None, 2859),
('Уфа', 2786, None, None, 2786),
('Казань', 2205, None, None, 2205),
('Владивосток', 2134, None, None, 2134),
]
parts = []
for (terr, level, mech, devs, permits, decls, houses, area, flats) in housing_summary:
parts.append(f"('{SNAP_APR26}', {esc(terr)}, '{level}', '{mech}', {integer(devs)}, "
f"{integer(permits)}, {integer(decls)}, {integer(houses)}, {num(area)}, {num(flats)})")
for breakdown, level in [(region_breakdown, 'region'), (city_breakdown, 'city')]:
for (terr, escrow, fund, no_attr, total) in breakdown:
for mech, area in [('escrow', escrow), ('fund', fund),
('no_attraction', no_attr), ('all', total)]:
if area is None: continue
parts.append(f"('{SNAP_APR26}', {esc(terr)}, '{level}', '{mech}', NULL, "
f"NULL, NULL, NULL, {num(area)}, NULL)")
psql(f"""
INSERT INTO domrf_housing_summary (snapshot_date, territory_name, territory_level,
mechanism, developers_count, permits_count, declarations_count, houses_count,
area_thousand_sqm, flats_thousand)
VALUES {','.join(parts)}
ON CONFLICT (snapshot_date, territory_name, mechanism) DO UPDATE SET
developers_count = EXCLUDED.developers_count,
permits_count = EXCLUDED.permits_count,
declarations_count = EXCLUDED.declarations_count,
houses_count = EXCLUDED.houses_count,
area_thousand_sqm = EXCLUDED.area_thousand_sqm,
flats_thousand = EXCLUDED.flats_thousand;
""")
print(f'housing_summary: {len(parts)}')
# planned_commissioning (year, escrow, fund, no_attr, total)
pc = [
(2026, 37084, 685, 154, 37923),
(2027, 34896, 246, 221, 35363),
(2028, 26360, 113, 30, 26503),
(2029, 10924, 19, 24, 10967),
(2030, 4245, 13, 32, 4290),
(2031, 1797, 28, None, 1825),
(2032, 1323, None, None, 1323),
(2033, 351, None, None, 351),
(2034, 234, None, None, 234),
(2035, 67, None, None, 67),
]
parts = [f"('{SNAP_APR26}', {y}, {num(e)}, {num(f)}, {num(n)}, {num(t)})"
for (y, e, f, n, t) in pc]
psql(f"""
INSERT INTO domrf_planned_commissioning (snapshot_date, plan_year, escrow_th_sqm,
fund_th_sqm, no_attraction_th_sqm, total_th_sqm)
VALUES {','.join(parts)}
ON CONFLICT (snapshot_date, plan_year) DO UPDATE SET
escrow_th_sqm = EXCLUDED.escrow_th_sqm,
fund_th_sqm = EXCLUDED.fund_th_sqm,
no_attraction_th_sqm = EXCLUDED.no_attraction_th_sqm,
total_th_sqm = EXCLUDED.total_th_sqm;
""")
print(f'planned_commissioning: {len(parts)}')
# ─────────── 5. sold_out_info.pdf → 3 tables ───────────
# RF total
psql(f"""
INSERT INTO domrf_sold_out (snapshot_date, territory_name, total_area_th_sqm,
sold_area_th_sqm, unsold_area_th_sqm, not_open_area_th_sqm,
sold_pct, unsold_pct, not_open_pct, avg_price_per_sqm, attracted_funds_mln_rub)
VALUES ('{SNAP_SOLD}', 'Российская Федерация',
118876, 36749, 53044, 29084, 31, 45, 24, 216316, 7949316)
ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
total_area_th_sqm = EXCLUDED.total_area_th_sqm,
sold_area_th_sqm = EXCLUDED.sold_area_th_sqm,
unsold_area_th_sqm = EXCLUDED.unsold_area_th_sqm,
not_open_area_th_sqm = EXCLUDED.not_open_area_th_sqm,
sold_pct = EXCLUDED.sold_pct,
unsold_pct = EXCLUDED.unsold_pct,
not_open_pct = EXCLUDED.not_open_pct,
avg_price_per_sqm = EXCLUDED.avg_price_per_sqm,
attracted_funds_mln_rub = EXCLUDED.attracted_funds_mln_rub;
""")
# Top-8 regions: (region, total, sold%, unsold%, not_open%)
so_regions = [
('Город Москва', 16405, 47, 44, None),
('Краснодарский край', 8286, 20, 39, 41),
('Московская область', 7942, 37, 43, None),
('Свердловская область', 5686, 29, 54, None),
('Город Санкт-Петербург', 5336, 40, 44, None),
('Ростовская область', 4990, None, 48, None),
('Ленинградская область', 4705, None, 44, None),
('Тюменская область', 3573, None, 50, None),
]
parts = []
for (r, total, sp, up, nop) in so_regions:
parts.append(f"('{SNAP_SOLD}', {esc(r)}, {num(total)}, NULL, NULL, NULL, "
f"{num(sp)}, {num(up)}, {num(nop)}, NULL, NULL)")
psql(f"""
INSERT INTO domrf_sold_out (snapshot_date, territory_name, total_area_th_sqm,
sold_area_th_sqm, unsold_area_th_sqm, not_open_area_th_sqm,
sold_pct, unsold_pct, not_open_pct, avg_price_per_sqm, attracted_funds_mln_rub)
VALUES {','.join(parts)}
ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
total_area_th_sqm = EXCLUDED.total_area_th_sqm,
sold_pct = EXCLUDED.sold_pct,
unsold_pct = EXCLUDED.unsold_pct,
not_open_pct = EXCLUDED.not_open_pct;
""")
print(f'sold_out: 1 RF + {len(parts)} regions')
# by_year — (year, total_th_sqm, pct, sold%, unsold%, not_open%)
so_year = [
('2026', 40201, 34, 52, 38, 10),
('2027', 34880, 29, 30, 53, 17),
('2028', 25363, 21, 17, 52, 31),
('2029', 10359, 9, None, 37, 54),
('2030', 4127, 3, None, None, 71),
('2031+', 3946, 3, None, None, 65),
]
parts = [f"('{SNAP_SOLD}', '{y}', {num(t)}, {num(p)}, {num(s)}, {num(u)}, {num(n)})"
for (y, t, p, s, u, n) in so_year]
psql(f"""
INSERT INTO domrf_sold_out_by_year (snapshot_date, plan_year, total_th_sqm,
pct_of_total, sold_pct, unsold_pct, not_open_pct)
VALUES {','.join(parts)}
ON CONFLICT (snapshot_date, plan_year) DO UPDATE SET
total_th_sqm = EXCLUDED.total_th_sqm,
pct_of_total = EXCLUDED.pct_of_total,
sold_pct = EXCLUDED.sold_pct,
unsold_pct = EXCLUDED.unsold_pct,
not_open_pct = EXCLUDED.not_open_pct;
""")
print(f'sold_out_by_year: {len(parts)}')
# by_progress
so_prog = [
('not_open', 29084, 24, None, 100),
('0-20', 26170, 22, 9, 91),
('21-40', 21790, 18, 30, 70),
('41-60', 17409, 15, 50, 50),
('61-80', 13402, 11, 69, 31),
('80+', 11021, 9, 90, 9),
]
parts = [f"('{SNAP_SOLD}', '{b}', {num(t)}, {num(p)}, {num(s)}, {num(u)})"
for (b, t, p, s, u) in so_prog]
psql(f"""
INSERT INTO domrf_sold_out_by_progress (snapshot_date, progress_bucket, total_th_sqm,
pct_of_total, sold_pct_within, unsold_pct_within)
VALUES {','.join(parts)}
ON CONFLICT (snapshot_date, progress_bucket) DO UPDATE SET
total_th_sqm = EXCLUDED.total_th_sqm,
pct_of_total = EXCLUDED.pct_of_total,
sold_pct_within = EXCLUDED.sold_pct_within,
unsold_pct_within = EXCLUDED.unsold_pct_within;
""")
print(f'sold_out_by_progress: {len(parts)}')
# ─────────── 6. mortgage_rates.pdf → domrf_mortgage_rates ───────────
rates = [
# (bank, primary, secondary, refi, note)
('Сбербанк', 20.2, 19.7, 21.5, None),
('ВТБ', 19.9, 19.9, 19.6, None),
('Банк ДОМ.РФ', 18.2, 18.2, 22.1, None),
('Альфа-банк', 19.49, 19.49, 18.99, None),
('Совкомбанк', 19.99, 20.49, 19.99, None),
('Т-Банк', 16.9, 16.9, 16.9, None),
('Банк Санкт-Петербург', 18.49, 18.49, 18.49, None),
('Промсвязьбанк', 19.49, 18.79, 18.99, None),
('Московский Кредитный Банк', 18.6, 18.6, 18.6, None),
('Уралсиб', 17.89, 17.89, 18.99, None),
('Абсолют банк', 19.35, 19.35, 19.35, None),
('Металлинвестбанк', 18.1, 18.1, 18.5, 'к пред.неделе -0.7 пп'),
('Газпромбанк', None, None, None, 'Максимальный срок 19 лет, рефинанс не предоставляет'),
('Азиатско-Тихоокеанский Банк', 18.9, 18.9, 18.9, None),
('Россельхозбанк', None, None, None, 'не предоставляет'),
('Банк Кубань Кредит', 16.4, 16.4, 15.9, None),
('Транскапиталбанк', 19.6, 19.6, 19.6, None),
('УБРиР', None, None, None, 'не предоставляет'),
('Ак Барс Банк', None, None, None, 'ПВ от 50%, рефинанс не предоставляет'),
('Средневзвешенная', 19.90, 19.59, 18.81, 'aggregate top-20'),
]
parts = [f"('{SNAP_MORTGAGE}', {esc(b)}, {num(p)}, {num(s)}, {num(r)}, {esc(n)})"
for (b, p, s, r, n) in rates]
psql(f"""
INSERT INTO domrf_mortgage_rates (snapshot_date, bank_name, primary_rate, secondary_rate, refinance_rate, note)
VALUES {','.join(parts)}
ON CONFLICT (snapshot_date, bank_name) DO UPDATE SET
primary_rate = EXCLUDED.primary_rate,
secondary_rate = EXCLUDED.secondary_rate,
refinance_rate = EXCLUDED.refinance_rate,
note = EXCLUDED.note;
""")
print(f'mortgage_rates: {len(parts)}')
# Summary
print('\n--- summary ---')
out = psql("""
SELECT 'developers_full' tab, COUNT(*) FROM domrf_developers_full
UNION ALL SELECT 'escrow_banks', COUNT(*) FROM domrf_escrow_banks
UNION ALL SELECT 'guaranty_regions', COUNT(*) FROM domrf_guaranty_regions
UNION ALL SELECT 'housing_summary', COUNT(*) FROM domrf_housing_summary
UNION ALL SELECT 'planned_commissioning', COUNT(*) FROM domrf_planned_commissioning
UNION ALL SELECT 'sold_out', COUNT(*) FROM domrf_sold_out
UNION ALL SELECT 'sold_out_by_year', COUNT(*) FROM domrf_sold_out_by_year
UNION ALL SELECT 'sold_out_by_progress', COUNT(*) FROM domrf_sold_out_by_progress
UNION ALL SELECT 'mortgage_rates', COUNT(*) FROM domrf_mortgage_rates
ORDER BY 1;
""")
print(out)
if __name__ == '__main__':
main()