add data
This commit is contained in:
lekss361
parent
a2dcd9844a
commit
c13cbaef2e
18 changed files with 2409 additions and 0 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -21,3 +21,9 @@ node_modules/
|
||||||
|
|
||||||
# Claude Code local state
|
# Claude Code local state
|
||||||
.claude/
|
.claude/
|
||||||
|
|
||||||
|
# Local data (raw CSVs from Rosreestr/etc — too large for git)
|
||||||
|
data/raw/
|
||||||
|
*.csv
|
||||||
|
*.csv.zip
|
||||||
|
!backend/db/init/*.sql
|
||||||
|
|
|
||||||
92
data/sql/01_schema_rosreestr_deals.sql
Normal file
92
data/sql/01_schema_rosreestr_deals.sql
Normal file
|
|
@ -0,0 +1,92 @@
|
||||||
|
-- Rosreestr open dataset: dataset_СДЕЛКИ (registered real estate deals).
|
||||||
|
-- Source: https://rosreestr.gov.ru/data-sets/{Q}%20квартал%20{YYYY}г./dataset_СДЕЛКИ_r-r_01-92_y_{YYYY}_q_{Q}.csv.zip
|
||||||
|
-- Encoding: UTF-8. Separator: '~' (since 2024 Q4) or ';' (2024 Q3 only).
|
||||||
|
-- Granularity: aggregated rows — same (region+district+city+cad_quarter+street+type+material+year+floor+purpose+area+doc_type) deals
|
||||||
|
-- collapsed into a single row with `deal_count` (was `number`) and AVERAGE price.
|
||||||
|
-- Time granularity: quarter (period_start_date = first day of quarter).
|
||||||
|
-- Coverage: ALL registered transactions (DKP=secondary+land+commercial, DDU=primary in МКД).
|
||||||
|
-- Excluded: gratuitous deals (gift/inheritance), buyer/seller PII, exact deal date, apartment number, room count, mortgage flag.
|
||||||
|
-- Data quality notes (observed in 7 loaded quarters):
|
||||||
|
-- * deal_price может быть мусорным с десятками знаков — фильтруем при INSERT
|
||||||
|
-- * year_build бывает с датой типа '09.07.1988' — оставляем NULL если не 4-цифровой год
|
||||||
|
-- * wall_material_code/realestate_type_code иногда '"code1;code2"' — поэтому TEXT вместо VARCHAR(15)
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS rosreestr_deals (
|
||||||
|
id BIGSERIAL,
|
||||||
|
-- Origin
|
||||||
|
source_quarter VARCHAR(8) NOT NULL, -- '2024Q3' for traceability
|
||||||
|
period_start_date DATE NOT NULL, -- first day of the quarter (partition key)
|
||||||
|
-- Geo
|
||||||
|
region_code SMALLINT NOT NULL, -- 66 = Свердловская обл, 77 = Москва, ...
|
||||||
|
okato TEXT, -- ОКАТО (often '-' or 11 digits)
|
||||||
|
district TEXT, -- муниципальный район/округ
|
||||||
|
city TEXT, -- населённый пункт (часто пустой)
|
||||||
|
quarter_cad_number VARCHAR(30) NOT NULL, -- '66:41:0506001' — гео-ключ для JOIN с ПКК / NSPD
|
||||||
|
street TEXT,
|
||||||
|
-- Object
|
||||||
|
realestate_type_code TEXT, -- 002001001000=ЗУ | 002001002000=Здание | 002001003000=Помещение | 002001009000=Машиноместо
|
||||||
|
wall_material_code TEXT, -- справочник 126_УНСИ (может быть multi-value)
|
||||||
|
purpose_code TEXT, -- 66/67/148_УНСИ
|
||||||
|
year_build SMALLINT,
|
||||||
|
floor TEXT,
|
||||||
|
area NUMERIC, -- м²
|
||||||
|
-- Deal
|
||||||
|
doc_type VARCHAR(8) NOT NULL, -- 'ДКП' | 'ДДУ'
|
||||||
|
deal_price NUMERIC, -- средняя если deal_count > 1
|
||||||
|
currency VARCHAR(20) DEFAULT 'рубль',
|
||||||
|
deal_count INT NOT NULL DEFAULT 1, -- было `number`
|
||||||
|
-- Computed
|
||||||
|
price_per_sqm NUMERIC GENERATED ALWAYS AS (
|
||||||
|
CASE WHEN area > 0 AND deal_price > 0 THEN deal_price / area ELSE NULL END
|
||||||
|
) STORED,
|
||||||
|
-- Audit
|
||||||
|
loaded_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
PRIMARY KEY (id, period_start_date)
|
||||||
|
) PARTITION BY RANGE (period_start_date);
|
||||||
|
|
||||||
|
-- Quarterly partitions for available data (2024 Q3 — 2026 Q1)
|
||||||
|
CREATE TABLE IF NOT EXISTS rosreestr_deals_2024q3 PARTITION OF rosreestr_deals
|
||||||
|
FOR VALUES FROM ('2024-07-01') TO ('2024-10-01');
|
||||||
|
CREATE TABLE IF NOT EXISTS rosreestr_deals_2024q4 PARTITION OF rosreestr_deals
|
||||||
|
FOR VALUES FROM ('2024-10-01') TO ('2025-01-01');
|
||||||
|
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q1 PARTITION OF rosreestr_deals
|
||||||
|
FOR VALUES FROM ('2025-01-01') TO ('2025-04-01');
|
||||||
|
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q2 PARTITION OF rosreestr_deals
|
||||||
|
FOR VALUES FROM ('2025-04-01') TO ('2025-07-01');
|
||||||
|
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q3 PARTITION OF rosreestr_deals
|
||||||
|
FOR VALUES FROM ('2025-07-01') TO ('2025-10-01');
|
||||||
|
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q4 PARTITION OF rosreestr_deals
|
||||||
|
FOR VALUES FROM ('2025-10-01') TO ('2026-01-01');
|
||||||
|
CREATE TABLE IF NOT EXISTS rosreestr_deals_2026q1 PARTITION OF rosreestr_deals
|
||||||
|
FOR VALUES FROM ('2026-01-01') TO ('2026-04-01');
|
||||||
|
|
||||||
|
-- Indexes
|
||||||
|
CREATE INDEX IF NOT EXISTS rosreestr_deals_region_cq_idx
|
||||||
|
ON rosreestr_deals (region_code, quarter_cad_number);
|
||||||
|
CREATE INDEX IF NOT EXISTS rosreestr_deals_period_brin
|
||||||
|
ON rosreestr_deals USING BRIN (period_start_date);
|
||||||
|
CREATE INDEX IF NOT EXISTS rosreestr_deals_dealtype_idx
|
||||||
|
ON rosreestr_deals (doc_type) WHERE doc_type IN ('ДКП', 'ДДУ');
|
||||||
|
CREATE INDEX IF NOT EXISTS rosreestr_deals_realtype_idx
|
||||||
|
ON rosreestr_deals (realestate_type_code);
|
||||||
|
|
||||||
|
-- Staging: один общий unlogged-буфер. Перед каждым кварталом TRUNCATE.
|
||||||
|
CREATE UNLOGGED TABLE IF NOT EXISTS rosreestr_deals_staging (
|
||||||
|
deal_count INT,
|
||||||
|
okato TEXT,
|
||||||
|
region_code SMALLINT,
|
||||||
|
district TEXT,
|
||||||
|
city TEXT,
|
||||||
|
quarter_cad_number TEXT,
|
||||||
|
street TEXT,
|
||||||
|
realestate_type_code TEXT,
|
||||||
|
wall_material_code TEXT,
|
||||||
|
year_build TEXT,
|
||||||
|
floor TEXT,
|
||||||
|
purpose_code TEXT,
|
||||||
|
area NUMERIC,
|
||||||
|
period_start_date DATE,
|
||||||
|
deal_price NUMERIC,
|
||||||
|
currency VARCHAR(20),
|
||||||
|
doc_type VARCHAR(8)
|
||||||
|
);
|
||||||
96
data/sql/02_load_all_quarters.sh
Normal file
96
data/sql/02_load_all_quarters.sh
Normal file
|
|
@ -0,0 +1,96 @@
|
||||||
|
#!/usr/bin/env bash
|
||||||
|
# Loads all 7 quarters of dataset_СДЕЛКИ into rosreestr_deals via staging.
|
||||||
|
# Q3 2024 uses ';' separator, all later quarters use '~'.
|
||||||
|
#
|
||||||
|
# Usage:
|
||||||
|
# bash 02_load_all_quarters.sh # local test container (gd_test_pg)
|
||||||
|
# PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... bash 02_load_all_quarters.sh # remote (via SSH tunnel)
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
if [[ -n "${PG_HOST:-}" ]]; then
|
||||||
|
# Remote / TCP mode (e.g. prod via SSH tunnel)
|
||||||
|
PG_PORT="${PG_PORT:-5432}"
|
||||||
|
PG_USER="${PG_USER:-gendesign}"
|
||||||
|
PG_DB="${PG_DB:-gendesign}"
|
||||||
|
PSQL="docker run --rm -i -e PGPASSWORD postgres:16-alpine psql -h host.docker.internal -p $PG_PORT -U $PG_USER -d $PG_DB -v ON_ERROR_STOP=1"
|
||||||
|
PSQL_PIPE="docker run --rm -i -e PGPASSWORD postgres:16-alpine psql -h host.docker.internal -p $PG_PORT -U $PG_USER -d $PG_DB -v ON_ERROR_STOP=1"
|
||||||
|
else
|
||||||
|
# Local docker exec into the test container
|
||||||
|
PG_CONTAINER="${PG_CONTAINER:-gd_test_pg}"
|
||||||
|
PSQL="docker exec -i $PG_CONTAINER psql -U gendesign -d gendesign -v ON_ERROR_STOP=1"
|
||||||
|
PSQL_PIPE="$PSQL"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cd "$(dirname "$0")/../raw"
|
||||||
|
|
||||||
|
declare -a JOBS=(
|
||||||
|
"2024Q3:2024-07-01:dataset_СДЕЛКИ_r_all_q_3.csv:;"
|
||||||
|
"2024Q4:2024-10-01:dataset_СДЕЛКИ_r-r_01-92_y_2024_q_4.csv:~"
|
||||||
|
"2025Q1:2025-01-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_1.csv:~"
|
||||||
|
"2025Q2:2025-04-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_2.csv:~"
|
||||||
|
"2025Q3:2025-07-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_3.csv:~"
|
||||||
|
"2025Q4:2025-10-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_4.csv:~"
|
||||||
|
"2026Q1:2026-01-01:dataset_СДЕЛКИ_r-r_01-92_y_2026_q_1.csv:~"
|
||||||
|
)
|
||||||
|
|
||||||
|
for job in "${JOBS[@]}"; do
|
||||||
|
IFS=':' read -r SRC_Q PERIOD FILE SEP <<< "$job"
|
||||||
|
echo "=== $SRC_Q ($FILE, sep='$SEP') ==="
|
||||||
|
$PSQL -c "TRUNCATE rosreestr_deals_staging;"
|
||||||
|
start=$(date +%s)
|
||||||
|
cat "$FILE" | $PSQL -c "\\copy rosreestr_deals_staging FROM stdin (FORMAT csv, HEADER true, DELIMITER '$SEP', QUOTE '\"', NULL '')"
|
||||||
|
$PSQL <<SQL
|
||||||
|
INSERT INTO rosreestr_deals (
|
||||||
|
source_quarter, period_start_date,
|
||||||
|
region_code, okato, district, city, quarter_cad_number, street,
|
||||||
|
realestate_type_code, wall_material_code, purpose_code,
|
||||||
|
year_build, floor, area,
|
||||||
|
doc_type, deal_price, currency, deal_count
|
||||||
|
)
|
||||||
|
SELECT
|
||||||
|
'$SRC_Q', period_start_date,
|
||||||
|
region_code, NULLIF(okato, '-'), NULLIF(district, ''), NULLIF(city, ''),
|
||||||
|
quarter_cad_number, NULLIF(street, ''),
|
||||||
|
NULLIF(realestate_type_code, ''),
|
||||||
|
NULLIF(wall_material_code, ''),
|
||||||
|
NULLIF(purpose_code, ''),
|
||||||
|
-- year_build can contain garbage like '09.07.1988' — keep only plausible 4-digit years
|
||||||
|
CASE WHEN year_build ~ '^[12][0-9]{3}$' THEN year_build::INT ELSE NULL END,
|
||||||
|
NULLIF(floor, ''),
|
||||||
|
-- discard absurd area/price (data-quality outliers from source)
|
||||||
|
CASE WHEN area > 0 AND area < 1e8 THEN area ELSE NULL END,
|
||||||
|
NULLIF(doc_type, ''),
|
||||||
|
CASE WHEN deal_price > 0 AND deal_price < 1e15 THEN deal_price ELSE NULL END,
|
||||||
|
COALESCE(NULLIF(currency, ''), 'рубль'),
|
||||||
|
COALESCE(deal_count, 1)
|
||||||
|
FROM rosreestr_deals_staging
|
||||||
|
WHERE region_code IS NOT NULL
|
||||||
|
AND quarter_cad_number IS NOT NULL
|
||||||
|
AND quarter_cad_number <> ''
|
||||||
|
AND doc_type IS NOT NULL
|
||||||
|
AND doc_type <> ''
|
||||||
|
AND period_start_date IS NOT NULL;
|
||||||
|
SQL
|
||||||
|
end=$(date +%s)
|
||||||
|
echo " loaded in $((end-start))s"
|
||||||
|
done
|
||||||
|
|
||||||
|
$PSQL <<'SQL'
|
||||||
|
SELECT
|
||||||
|
source_quarter,
|
||||||
|
COUNT(*) AS rows_loaded,
|
||||||
|
SUM(deal_count) AS deals,
|
||||||
|
SUM(deal_count) FILTER (WHERE region_code = 66) AS sverdl_deals,
|
||||||
|
pg_size_pretty(pg_relation_size('rosreestr_deals_'||lower(source_quarter))) AS partition_size
|
||||||
|
FROM rosreestr_deals
|
||||||
|
GROUP BY source_quarter
|
||||||
|
ORDER BY source_quarter;
|
||||||
|
|
||||||
|
SELECT
|
||||||
|
'TOTAL' AS quarter,
|
||||||
|
COUNT(*) AS rows,
|
||||||
|
SUM(deal_count) AS deals,
|
||||||
|
SUM(deal_count) FILTER (WHERE region_code = 66) AS sverdl_deals,
|
||||||
|
pg_size_pretty(pg_total_relation_size('rosreestr_deals')) AS total_size
|
||||||
|
FROM rosreestr_deals;
|
||||||
|
SQL
|
||||||
98
data/sql/03_materialized_views.sql
Normal file
98
data/sql/03_materialized_views.sql
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
-- Materialized aggregates for fast UI queries.
|
||||||
|
-- Refresh strategy: REFRESH MATERIALIZED VIEW CONCURRENTLY after each new quarter load.
|
||||||
|
|
||||||
|
-- 1. Per cadastral quarter × period × doc_type × area_bucket
|
||||||
|
DROP MATERIALIZED VIEW IF EXISTS rr_agg_cad_quarter CASCADE;
|
||||||
|
CREATE MATERIALIZED VIEW rr_agg_cad_quarter AS
|
||||||
|
SELECT
|
||||||
|
region_code,
|
||||||
|
quarter_cad_number,
|
||||||
|
period_start_date,
|
||||||
|
doc_type,
|
||||||
|
-- area bucket inspired by NSPD MarketAnalytics filter
|
||||||
|
CASE
|
||||||
|
WHEN area < 25 THEN 'lt25'
|
||||||
|
WHEN area < 40 THEN '25-40'
|
||||||
|
WHEN area < 60 THEN '40-60'
|
||||||
|
WHEN area < 90 THEN '60-90'
|
||||||
|
WHEN area < 150 THEN '90-150'
|
||||||
|
ELSE 'gt150'
|
||||||
|
END AS area_bucket,
|
||||||
|
SUM(deal_count) AS deals_count,
|
||||||
|
COUNT(*) AS rows_count,
|
||||||
|
SUM(deal_count * area) AS total_area_sqm,
|
||||||
|
SUM(deal_count * deal_price) AS total_value_rub,
|
||||||
|
percentile_cont(0.10) WITHIN GROUP (ORDER BY price_per_sqm) AS p10_price_sqm,
|
||||||
|
percentile_cont(0.50) WITHIN GROUP (ORDER BY price_per_sqm) AS median_price_sqm,
|
||||||
|
percentile_cont(0.90) WITHIN GROUP (ORDER BY price_per_sqm) AS p90_price_sqm,
|
||||||
|
AVG(price_per_sqm) AS mean_price_sqm,
|
||||||
|
MIN(price_per_sqm) AS min_price_sqm,
|
||||||
|
MAX(price_per_sqm) AS max_price_sqm
|
||||||
|
FROM rosreestr_deals
|
||||||
|
WHERE area BETWEEN 5 AND 1000
|
||||||
|
AND deal_price BETWEEN 100000 AND 1e10 -- discard outliers (typos, junk in source)
|
||||||
|
AND realestate_type_code IN ('002001003000') -- помещения (квартиры). Здания/ЗУ — отдельный rollup
|
||||||
|
GROUP BY region_code, quarter_cad_number, period_start_date, doc_type,
|
||||||
|
CASE WHEN area<25 THEN 'lt25' WHEN area<40 THEN '25-40' WHEN area<60 THEN '40-60'
|
||||||
|
WHEN area<90 THEN '60-90' WHEN area<150 THEN '90-150' ELSE 'gt150' END
|
||||||
|
WITH NO DATA;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX rr_agg_cq_pk ON rr_agg_cad_quarter
|
||||||
|
(region_code, quarter_cad_number, period_start_date, doc_type, area_bucket);
|
||||||
|
CREATE INDEX rr_agg_cq_period_idx ON rr_agg_cad_quarter (period_start_date);
|
||||||
|
CREATE INDEX rr_agg_cq_region_period_idx ON rr_agg_cad_quarter (region_code, period_start_date);
|
||||||
|
|
||||||
|
-- 2. Per region × period × doc_type — для регионального обзора (Урал dashboard)
|
||||||
|
DROP MATERIALIZED VIEW IF EXISTS rr_agg_region CASCADE;
|
||||||
|
CREATE MATERIALIZED VIEW rr_agg_region AS
|
||||||
|
SELECT
|
||||||
|
region_code,
|
||||||
|
period_start_date,
|
||||||
|
doc_type,
|
||||||
|
realestate_type_code,
|
||||||
|
SUM(deal_count) AS deals_count,
|
||||||
|
COUNT(*) AS rows_count,
|
||||||
|
SUM(deal_count * area) AS total_area_sqm,
|
||||||
|
SUM(deal_count * deal_price) AS total_value_rub,
|
||||||
|
percentile_cont(0.10) WITHIN GROUP (ORDER BY price_per_sqm) AS p10_price_sqm,
|
||||||
|
percentile_cont(0.50) WITHIN GROUP (ORDER BY price_per_sqm) AS median_price_sqm,
|
||||||
|
percentile_cont(0.90) WITHIN GROUP (ORDER BY price_per_sqm) AS p90_price_sqm
|
||||||
|
FROM rosreestr_deals
|
||||||
|
WHERE area BETWEEN 5 AND 1000
|
||||||
|
AND deal_price BETWEEN 100000 AND 1e10
|
||||||
|
GROUP BY region_code, period_start_date, doc_type, realestate_type_code
|
||||||
|
WITH NO DATA;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX rr_agg_region_pk ON rr_agg_region
|
||||||
|
(region_code, period_start_date, doc_type, realestate_type_code);
|
||||||
|
|
||||||
|
-- 3. Per city/settlement (для дашборда Екатеринбург)
|
||||||
|
DROP MATERIALIZED VIEW IF EXISTS rr_agg_settlement CASCADE;
|
||||||
|
CREATE MATERIALIZED VIEW rr_agg_settlement AS
|
||||||
|
SELECT
|
||||||
|
region_code,
|
||||||
|
COALESCE(city, district) AS settlement_name,
|
||||||
|
period_start_date,
|
||||||
|
doc_type,
|
||||||
|
realestate_type_code,
|
||||||
|
SUM(deal_count) AS deals_count,
|
||||||
|
SUM(deal_count * area) AS total_area_sqm,
|
||||||
|
percentile_cont(0.50) WITHIN GROUP (ORDER BY price_per_sqm) AS median_price_sqm,
|
||||||
|
percentile_cont(0.10) WITHIN GROUP (ORDER BY price_per_sqm) AS p10_price_sqm,
|
||||||
|
percentile_cont(0.90) WITHIN GROUP (ORDER BY price_per_sqm) AS p90_price_sqm,
|
||||||
|
COUNT(DISTINCT quarter_cad_number) AS distinct_cad_quarters
|
||||||
|
FROM rosreestr_deals
|
||||||
|
WHERE area BETWEEN 5 AND 1000
|
||||||
|
AND deal_price BETWEEN 100000 AND 1e10
|
||||||
|
AND COALESCE(city, district) IS NOT NULL
|
||||||
|
GROUP BY region_code, COALESCE(city, district), period_start_date, doc_type, realestate_type_code
|
||||||
|
WITH NO DATA;
|
||||||
|
|
||||||
|
CREATE UNIQUE INDEX rr_agg_settl_pk ON rr_agg_settlement
|
||||||
|
(region_code, settlement_name, period_start_date, doc_type, realestate_type_code);
|
||||||
|
CREATE INDEX rr_agg_settl_search ON rr_agg_settlement (region_code, settlement_name);
|
||||||
|
|
||||||
|
-- Initial fill
|
||||||
|
REFRESH MATERIALIZED VIEW rr_agg_cad_quarter;
|
||||||
|
REFRESH MATERIALIZED VIEW rr_agg_region;
|
||||||
|
REFRESH MATERIALIZED VIEW rr_agg_settlement;
|
||||||
67
data/sql/10_schema_domrf.sql
Normal file
67
data/sql/10_schema_domrf.sql
Normal file
|
|
@ -0,0 +1,67 @@
|
||||||
|
-- DOM.RF (наш.дом.рф) portal-analytics dashboard data.
|
||||||
|
-- Source: https://xn--80az8a.xn--d1aqf.xn--p1ai/portal-analytics/api/*
|
||||||
|
-- Coverage: aggregate of construction-stage flats in МКД (multi-apartment buildings).
|
||||||
|
-- ⚠ NOT historical: this is a SNAPSHOT of "currently being built" state. Updated continuously.
|
||||||
|
-- We store snapshot_date so we can rebuild trend over time by re-pulling.
|
||||||
|
|
||||||
|
-- 1. Federal districts + regions dictionary (from /dictionaries/regions)
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_regions (
|
||||||
|
region_id INT PRIMARY KEY,
|
||||||
|
region_name TEXT NOT NULL,
|
||||||
|
federal_district TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 2. Developers dictionary (from /dictionaries/developers, 2734 entries as of 2026-04-26)
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_developers (
|
||||||
|
developer_id TEXT PRIMARY KEY, -- '6208_0' (PRINZIP), '5655_0' (ПИК), '0_14583' etc.
|
||||||
|
developer_name TEXT NOT NULL
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 3. Snapshot metadata
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_snapshots (
|
||||||
|
snapshot_date DATE PRIMARY KEY,
|
||||||
|
fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
notes TEXT
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 4. By-region aggregates with breakdown by room count
|
||||||
|
-- One row per (region, room_count_type, snapshot)
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_region_aggregates (
|
||||||
|
region_id INT NOT NULL,
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
room_count_type TEXT NOT NULL, -- 'ONE'|'TWO'|'THREE'|'FOUR' or 'TOTAL' for region totals
|
||||||
|
flat_count INT NOT NULL,
|
||||||
|
area_sqm NUMERIC(14, 2) NOT NULL,
|
||||||
|
percent SMALLINT, -- % within region for room_count breakdown
|
||||||
|
PRIMARY KEY (region_id, snapshot_date, room_count_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 5. By-flat-area distribution (RF-wide, не работает фильтр по региону)
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_flat_area_distribution (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
region_id INT NOT NULL DEFAULT 0, -- 0 = вся РФ (api ignores ?regionId on this page)
|
||||||
|
area_bucket TEXT NOT NULL, -- 'FROM_0_TO_25', 'FROM_25_TO_35', ..., 'FROM_100'
|
||||||
|
room_count_type TEXT NOT NULL, -- 'ONE'|'TWO'|'THREE'|'FOUR' or 'TOTAL'
|
||||||
|
flat_count INT NOT NULL,
|
||||||
|
area_sqm NUMERIC(14, 2) NOT NULL,
|
||||||
|
percent SMALLINT,
|
||||||
|
PRIMARY KEY (snapshot_date, region_id, area_bucket, room_count_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 6. By-developer aggregates with breakdown by room count
|
||||||
|
-- Topology: developer × snapshot × room_count_type
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_developer_aggregates (
|
||||||
|
developer_id TEXT NOT NULL,
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
region_id INT NOT NULL DEFAULT 0, -- 0 = РФ агрегат (api ignores ?regionId on this page)
|
||||||
|
room_count_type TEXT NOT NULL,
|
||||||
|
flat_count INT NOT NULL,
|
||||||
|
area_sqm NUMERIC(14, 2) NOT NULL,
|
||||||
|
percent SMALLINT,
|
||||||
|
PRIMARY KEY (developer_id, snapshot_date, region_id, room_count_type)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Indexes
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_domrf_region_aggregates_room ON domrf_region_aggregates (room_count_type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_domrf_developer_aggregates_room ON domrf_developer_aggregates (room_count_type);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_domrf_developer_aggregates_dev ON domrf_developer_aggregates (developer_id);
|
||||||
183
data/sql/11_load_domrf.py
Normal file
183
data/sql/11_load_domrf.py
Normal file
|
|
@ -0,0 +1,183 @@
|
||||||
|
"""Loads DOM.RF analytics JSON files from data/raw/domrf/ into Postgres.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... python data/sql/11_load_domrf.py
|
||||||
|
# or for local test:
|
||||||
|
PGPASSWORD=test PG_PORT=6543 python data/sql/11_load_domrf.py
|
||||||
|
|
||||||
|
Files expected (snapshot from https://наш.дом.рф/portal-analytics/api/):
|
||||||
|
data/raw/domrf/regions.json — list of FO with regions inside
|
||||||
|
data/raw/domrf/developers.json — flat list of {id, name}
|
||||||
|
data/raw/domrf/by-region.json — per-region aggregates with room-count details
|
||||||
|
data/raw/domrf/by-flat-area-RF.json — RF area distribution with room-count details
|
||||||
|
data/raw/domrf/by-developer-RF.json — RF top developers with room-count details
|
||||||
|
data/raw/domrf/by-room-count-RF.json — RF totals (sanity check, not stored)
|
||||||
|
"""
|
||||||
|
import json, os, sys, datetime, subprocess
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
RAW = os.path.join(HERE, '..', 'raw', 'domrf')
|
||||||
|
|
||||||
|
PG_HOST = os.environ.get('PG_HOST', '127.0.0.1')
|
||||||
|
PG_PORT = os.environ.get('PG_PORT', '5432')
|
||||||
|
PG_USER = os.environ.get('PG_USER', 'gendesign')
|
||||||
|
PG_DB = os.environ.get('PG_DB', 'gendesign')
|
||||||
|
PG_PASS = os.environ.get('PGPASSWORD', '')
|
||||||
|
|
||||||
|
SNAPSHOT_DATE = os.environ.get('SNAPSHOT_DATE', datetime.date.today().isoformat())
|
||||||
|
|
||||||
|
|
||||||
|
def psql(sql_or_stdin, stdin_text=None):
|
||||||
|
"""Run psql via docker. Pipe SQL via stdin."""
|
||||||
|
cmd = [
|
||||||
|
'docker', 'run', '--rm', '-i',
|
||||||
|
'-e', f'PGPASSWORD={PG_PASS}',
|
||||||
|
'postgres:16-alpine',
|
||||||
|
'psql', '-h', 'host.docker.internal', '-p', PG_PORT, '-U', PG_USER, '-d', PG_DB,
|
||||||
|
'-v', 'ON_ERROR_STOP=1', '--quiet',
|
||||||
|
]
|
||||||
|
if stdin_text is None:
|
||||||
|
stdin_text = sql_or_stdin
|
||||||
|
res = subprocess.run(cmd, input=stdin_text, capture_output=True, text=True, encoding='utf-8')
|
||||||
|
if res.returncode != 0:
|
||||||
|
print('psql FAILED:', res.stderr, file=sys.stderr)
|
||||||
|
raise SystemExit(res.returncode)
|
||||||
|
return res.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def sql_escape(s):
|
||||||
|
return s.replace("'", "''")
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
raw_files = {
|
||||||
|
'regions': os.path.join(RAW, 'regions.json'),
|
||||||
|
'developers': os.path.join(RAW, 'developers.json'),
|
||||||
|
'by-region': os.path.join(RAW, 'by-region.json'),
|
||||||
|
'by-flat-area-RF': os.path.join(RAW, 'by-flat-area-RF.json'),
|
||||||
|
'by-developer-RF': os.path.join(RAW, 'by-developer-RF.json'),
|
||||||
|
}
|
||||||
|
for k, p in raw_files.items():
|
||||||
|
if not os.path.exists(p):
|
||||||
|
print(f'MISSING: {p}', file=sys.stderr); raise SystemExit(2)
|
||||||
|
|
||||||
|
# 0. Apply schema (idempotent CREATE TABLE IF NOT EXISTS)
|
||||||
|
schema = open(os.path.join(HERE, '10_schema_domrf.sql'), encoding='utf-8').read()
|
||||||
|
psql(schema)
|
||||||
|
print('schema applied')
|
||||||
|
|
||||||
|
# 1. Snapshot row
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_snapshots (snapshot_date, notes)
|
||||||
|
VALUES ('{SNAPSHOT_DATE}', 'naash.dom.rf portal-analytics, captured via chrome-devtools MCP')
|
||||||
|
ON CONFLICT (snapshot_date) DO UPDATE SET fetched_at = NOW();
|
||||||
|
""")
|
||||||
|
|
||||||
|
# 2. Regions dictionary
|
||||||
|
regions_data = json.load(open(raw_files['regions'], encoding='utf-8'))
|
||||||
|
parts = []
|
||||||
|
for fo in regions_data:
|
||||||
|
for r in fo['regions']:
|
||||||
|
parts.append(f"({r['id']}, '{sql_escape(r['name'])}', '{sql_escape(fo['name'])}')")
|
||||||
|
if parts:
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_regions (region_id, region_name, federal_district)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (region_id) DO UPDATE SET
|
||||||
|
region_name = EXCLUDED.region_name,
|
||||||
|
federal_district = EXCLUDED.federal_district;
|
||||||
|
""")
|
||||||
|
print(f'regions: {sum(len(fo["regions"]) for fo in regions_data)}')
|
||||||
|
|
||||||
|
# 3. Developers dictionary
|
||||||
|
devs = json.load(open(raw_files['developers'], encoding='utf-8'))
|
||||||
|
# batch insert in chunks of 500 to keep statement size sane
|
||||||
|
BATCH = 500
|
||||||
|
for i in range(0, len(devs), BATCH):
|
||||||
|
chunk = devs[i:i+BATCH]
|
||||||
|
parts = [f"('{sql_escape(d['id'])}', '{sql_escape(d['name'])}')" for d in chunk]
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_developers (developer_id, developer_name)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (developer_id) DO UPDATE SET developer_name = EXCLUDED.developer_name;
|
||||||
|
""")
|
||||||
|
print(f'developers: {len(devs)}')
|
||||||
|
|
||||||
|
# 4. By-region aggregates: insert TOTAL row + per-room rows for each region
|
||||||
|
byregion = json.load(open(raw_files['by-region'], encoding='utf-8'))
|
||||||
|
rows = []
|
||||||
|
for r in byregion:
|
||||||
|
rid = r['id']
|
||||||
|
# TOTAL row
|
||||||
|
rows.append((rid, SNAPSHOT_DATE, 'TOTAL', r['flatCount'], r['area'], None))
|
||||||
|
for d in r.get('details', []):
|
||||||
|
rows.append((rid, SNAPSHOT_DATE, d['roomCountType'], d['flatCount'], d['area'], d.get('percent')))
|
||||||
|
parts = [f"({rid}, '{sd}', '{rt}', {fc}, {a}, {'NULL' if p is None else p})"
|
||||||
|
for (rid, sd, rt, fc, a, p) in rows]
|
||||||
|
for i in range(0, len(parts), BATCH):
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_region_aggregates (region_id, snapshot_date, room_count_type, flat_count, area_sqm, percent)
|
||||||
|
VALUES {','.join(parts[i:i+BATCH])}
|
||||||
|
ON CONFLICT (region_id, snapshot_date, room_count_type) DO UPDATE SET
|
||||||
|
flat_count = EXCLUDED.flat_count,
|
||||||
|
area_sqm = EXCLUDED.area_sqm,
|
||||||
|
percent = EXCLUDED.percent;
|
||||||
|
""")
|
||||||
|
print(f'region_aggregates rows: {len(rows)}')
|
||||||
|
|
||||||
|
# 5. Flat-area distribution (RF-wide)
|
||||||
|
byarea = json.load(open(raw_files['by-flat-area-RF'], encoding='utf-8'))
|
||||||
|
rows = []
|
||||||
|
for bucket in byarea:
|
||||||
|
bid = bucket['id']
|
||||||
|
rows.append((bid, 'TOTAL', bucket['flatCount'], bucket['area'], bucket.get('percent')))
|
||||||
|
for d in bucket.get('details', []):
|
||||||
|
rows.append((bid, d['roomCountType'], d['flatCount'], d['area'], d.get('percent')))
|
||||||
|
parts = [f"('{SNAPSHOT_DATE}', 0, '{sql_escape(bid)}', '{rt}', {fc}, {a}, {'NULL' if p is None else p})"
|
||||||
|
for (bid, rt, fc, a, p) in rows]
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_flat_area_distribution (snapshot_date, region_id, area_bucket, room_count_type, flat_count, area_sqm, percent)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, region_id, area_bucket, room_count_type) DO UPDATE SET
|
||||||
|
flat_count = EXCLUDED.flat_count,
|
||||||
|
area_sqm = EXCLUDED.area_sqm,
|
||||||
|
percent = EXCLUDED.percent;
|
||||||
|
""")
|
||||||
|
print(f'flat_area_distribution rows: {len(rows)}')
|
||||||
|
|
||||||
|
# 6. By-developer aggregates (RF-wide because ?regionId is ignored)
|
||||||
|
bydev = json.load(open(raw_files['by-developer-RF'], encoding='utf-8'))
|
||||||
|
rows = []
|
||||||
|
for dev in bydev:
|
||||||
|
did = dev['id']
|
||||||
|
rows.append((did, 'TOTAL', dev['flatCount'], dev['area'], dev.get('percent')))
|
||||||
|
for d in dev.get('details', []):
|
||||||
|
rows.append((did, d['roomCountType'], d['flatCount'], d['area'], d.get('percent')))
|
||||||
|
parts = [f"('{sql_escape(did)}', '{SNAPSHOT_DATE}', 0, '{rt}', {fc}, {a}, {'NULL' if p is None else p})"
|
||||||
|
for (did, rt, fc, a, p) in rows]
|
||||||
|
for i in range(0, len(parts), BATCH):
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_developer_aggregates (developer_id, snapshot_date, region_id, room_count_type, flat_count, area_sqm, percent)
|
||||||
|
VALUES {','.join(parts[i:i+BATCH])}
|
||||||
|
ON CONFLICT (developer_id, snapshot_date, region_id, room_count_type) DO UPDATE SET
|
||||||
|
flat_count = EXCLUDED.flat_count,
|
||||||
|
area_sqm = EXCLUDED.area_sqm,
|
||||||
|
percent = EXCLUDED.percent;
|
||||||
|
""")
|
||||||
|
print(f'developer_aggregates rows: {len(rows)}')
|
||||||
|
|
||||||
|
# Final summary
|
||||||
|
out = psql("""
|
||||||
|
SELECT
|
||||||
|
(SELECT COUNT(*) FROM domrf_regions) AS regions,
|
||||||
|
(SELECT COUNT(*) FROM domrf_developers) AS developers,
|
||||||
|
(SELECT COUNT(*) FROM domrf_region_aggregates) AS region_agg,
|
||||||
|
(SELECT COUNT(*) FROM domrf_flat_area_distribution) AS area_dist,
|
||||||
|
(SELECT COUNT(*) FROM domrf_developer_aggregates) AS dev_agg;
|
||||||
|
""")
|
||||||
|
print('--- summary ---')
|
||||||
|
print(out)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
136
data/sql/20_schema_domrf_extras.sql
Normal file
136
data/sql/20_schema_domrf_extras.sql
Normal file
|
|
@ -0,0 +1,136 @@
|
||||||
|
-- DOM.RF additional snapshots (downloaded as XLSX/PDF from наш.дом.рф via "Скачать" buttons).
|
||||||
|
-- Snapshot date stamped on the file: 26.04.2026 (mortgage_rates is 19.04.2026).
|
||||||
|
-- These extend the JSON dashboard data with: full developer registry, escrow banks,
|
||||||
|
-- 214-FZ guaranty per region, sold-out (Tile3 velocity), housing info by mechanism, mortgage rates.
|
||||||
|
|
||||||
|
-- 1. Полный реестр застройщиков с метриками (НЕ группа компаний — а юрлица из проектных деклараций)
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_developers_full (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
developer_name TEXT NOT NULL,
|
||||||
|
area_thousand_sqm NUMERIC,
|
||||||
|
permits_count INT,
|
||||||
|
houses_count INT,
|
||||||
|
flats_count INT,
|
||||||
|
market_share_pct NUMERIC,
|
||||||
|
PRIMARY KEY (snapshot_date, developer_name)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 2. Банки эскроу с метриками
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_escrow_banks (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
bank_name TEXT NOT NULL,
|
||||||
|
houses_count INT,
|
||||||
|
permits_count INT,
|
||||||
|
area_thousand_sqm NUMERIC,
|
||||||
|
developers_count INT,
|
||||||
|
regions_count INT,
|
||||||
|
PRIMARY KEY (snapshot_date, bank_name)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 3. Долевое стр-во по 214-ФЗ — по регионам с разбивкой эскроу/ПП480
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_guaranty_regions (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
territory_name TEXT NOT NULL, -- 'Российская Федерация' | 'Центральный ФО' | 'Свердловская область' ...
|
||||||
|
territory_level TEXT NOT NULL, -- 'rf' | 'fo' | 'region'
|
||||||
|
-- Всего строится
|
||||||
|
total_permits INT,
|
||||||
|
total_area_th_sqm NUMERIC,
|
||||||
|
total_developers INT,
|
||||||
|
-- Имеют право привлекать ДУ по 214-ФЗ (всего)
|
||||||
|
fz214_permits INT,
|
||||||
|
fz214_area_th_sqm NUMERIC,
|
||||||
|
fz214_developers INT,
|
||||||
|
fz214_pct_of_total NUMERIC,
|
||||||
|
-- В т.ч. с эскроу
|
||||||
|
escrow_permits INT,
|
||||||
|
escrow_area_th_sqm NUMERIC,
|
||||||
|
escrow_developers INT,
|
||||||
|
-- В т.ч. ПП №480 (без эскроу)
|
||||||
|
pp480_permits INT,
|
||||||
|
pp480_area_th_sqm NUMERIC,
|
||||||
|
pp480_developers INT,
|
||||||
|
PRIMARY KEY (snapshot_date, territory_name)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 4. Сводные показатели по механизмам привлечения (housing_info)
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_housing_summary (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
territory_name TEXT NOT NULL, -- 'Российская Федерация' | город | регион
|
||||||
|
territory_level TEXT NOT NULL, -- 'rf' | 'region' | 'city'
|
||||||
|
mechanism TEXT NOT NULL, -- 'all' | 'escrow' | 'fund' | 'no_attraction'
|
||||||
|
developers_count INT,
|
||||||
|
permits_count INT,
|
||||||
|
declarations_count INT,
|
||||||
|
houses_count INT,
|
||||||
|
area_thousand_sqm NUMERIC,
|
||||||
|
flats_thousand NUMERIC,
|
||||||
|
PRIMARY KEY (snapshot_date, territory_name, mechanism)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 5. Плановые сроки ввода (housing_info, нижняя таблица)
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_planned_commissioning (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
plan_year INT NOT NULL,
|
||||||
|
escrow_th_sqm NUMERIC,
|
||||||
|
fund_th_sqm NUMERIC,
|
||||||
|
no_attraction_th_sqm NUMERIC,
|
||||||
|
total_th_sqm NUMERIC,
|
||||||
|
PRIMARY KEY (snapshot_date, plan_year)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 6. Реализация (sold_out_info) — РФ итог + регионы
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_sold_out (
|
||||||
|
snapshot_date DATE NOT NULL, -- '2026-04-11' (отчёт за март 2026)
|
||||||
|
territory_name TEXT NOT NULL,
|
||||||
|
-- Площади
|
||||||
|
total_area_th_sqm NUMERIC,
|
||||||
|
sold_area_th_sqm NUMERIC,
|
||||||
|
unsold_area_th_sqm NUMERIC,
|
||||||
|
not_open_area_th_sqm NUMERIC,
|
||||||
|
-- Проценты
|
||||||
|
sold_pct NUMERIC,
|
||||||
|
unsold_pct NUMERIC,
|
||||||
|
not_open_pct NUMERIC,
|
||||||
|
-- Дополнительно (только для РФ)
|
||||||
|
avg_price_per_sqm NUMERIC, -- ₽/м²
|
||||||
|
attracted_funds_mln_rub NUMERIC, -- млн ₽
|
||||||
|
PRIMARY KEY (snapshot_date, territory_name)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 7. Реализация по году ввода
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_sold_out_by_year (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
plan_year TEXT NOT NULL, -- '2026' .. '2031+'
|
||||||
|
total_th_sqm NUMERIC,
|
||||||
|
pct_of_total NUMERIC,
|
||||||
|
sold_pct NUMERIC,
|
||||||
|
unsold_pct NUMERIC,
|
||||||
|
not_open_pct NUMERIC,
|
||||||
|
PRIMARY KEY (snapshot_date, plan_year)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 8. Распределение по % реализации в доме
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_sold_out_by_progress (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
progress_bucket TEXT NOT NULL, -- 'not_open' | '0-20' | '21-40' | '41-60' | '61-80' | '80+'
|
||||||
|
total_th_sqm NUMERIC,
|
||||||
|
pct_of_total NUMERIC,
|
||||||
|
sold_pct_within NUMERIC,
|
||||||
|
unsold_pct_within NUMERIC,
|
||||||
|
PRIMARY KEY (snapshot_date, progress_bucket)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- 9. Ставки ипотеки топ-20 банков (mortgage_rates)
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_mortgage_rates (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
bank_name TEXT NOT NULL,
|
||||||
|
primary_rate NUMERIC, -- %, NULL если 'не предоставляет'
|
||||||
|
secondary_rate NUMERIC,
|
||||||
|
refinance_rate NUMERIC,
|
||||||
|
note TEXT, -- 'Максимальный срок – 19 лет', 'ПВ от 50%' etc.
|
||||||
|
PRIMARY KEY (snapshot_date, bank_name)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_domrf_developers_full_share ON domrf_developers_full (market_share_pct DESC);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_domrf_guaranty_level ON domrf_guaranty_regions (territory_level);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_domrf_housing_summary_terr ON domrf_housing_summary (territory_name, mechanism);
|
||||||
405
data/sql/21_load_domrf_extras.py
Normal file
405
data/sql/21_load_domrf_extras.py
Normal file
|
|
@ -0,0 +1,405 @@
|
||||||
|
"""Loads DOM.RF extra snapshots (XLSX downloads + PDF-extracted data) into Postgres.
|
||||||
|
|
||||||
|
Inputs in C:\\Users\\user\\Downloads\\:
|
||||||
|
- developer_rf_26-04-2026.xlsx → domrf_developers_full
|
||||||
|
- escrow_rf_26-04-2026.xlsx → domrf_escrow_banks
|
||||||
|
- guaranty_26-04-2026.xlsx → domrf_guaranty_regions
|
||||||
|
- housing_info_26-04-2026.pdf → domrf_housing_summary, domrf_planned_commissioning (hardcoded extract)
|
||||||
|
- sold_out_info_26-04-2026.pdf → domrf_sold_out, ..._by_year, ..._by_progress (hardcoded extract)
|
||||||
|
- mortgage_rates_19-04-2026.pdf → domrf_mortgage_rates (hardcoded extract)
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... python data/sql/21_load_domrf_extras.py
|
||||||
|
"""
|
||||||
|
import os, sys, subprocess
|
||||||
|
import openpyxl
|
||||||
|
|
||||||
|
DOWNLOADS = r'C:\Users\user\Downloads'
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
PG_PORT = os.environ.get('PG_PORT', '5432')
|
||||||
|
PG_USER = os.environ.get('PG_USER', 'gendesign')
|
||||||
|
PG_DB = os.environ.get('PG_DB', 'gendesign')
|
||||||
|
PG_PASS = os.environ.get('PGPASSWORD', '')
|
||||||
|
|
||||||
|
|
||||||
|
def psql(sql):
|
||||||
|
cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
|
||||||
|
'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT,
|
||||||
|
'-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet']
|
||||||
|
res = subprocess.run(cmd, input=sql, capture_output=True, text=True, encoding='utf-8')
|
||||||
|
if res.returncode != 0:
|
||||||
|
print('psql FAILED:', res.stderr[-2000:], file=sys.stderr)
|
||||||
|
raise SystemExit(res.returncode)
|
||||||
|
return res.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def esc(s):
|
||||||
|
if s is None: return 'NULL'
|
||||||
|
return "'" + str(s).replace("'", "''") + "'"
|
||||||
|
|
||||||
|
def num(v):
|
||||||
|
if v is None or v == '-' or v == '': return 'NULL'
|
||||||
|
try: return str(float(v))
|
||||||
|
except (ValueError, TypeError): return 'NULL'
|
||||||
|
|
||||||
|
def integer(v):
|
||||||
|
if v is None or v == '-' or v == '': return 'NULL'
|
||||||
|
try: return str(int(float(v)))
|
||||||
|
except (ValueError, TypeError): return 'NULL'
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
# 0. Apply schema
|
||||||
|
schema = open(os.path.join(HERE, '20_schema_domrf_extras.sql'), encoding='utf-8').read()
|
||||||
|
psql(schema)
|
||||||
|
print('schema applied')
|
||||||
|
|
||||||
|
SNAP_APR26 = '2026-04-26'
|
||||||
|
SNAP_MORTGAGE = '2026-04-19'
|
||||||
|
SNAP_SOLD = '2026-04-11' # отчётная дата на самом PDF
|
||||||
|
|
||||||
|
# ─────────── 1. developer_rf.xlsx → domrf_developers_full ───────────
|
||||||
|
wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'developer_rf_26-04-2026.xlsx'),
|
||||||
|
read_only=True, data_only=True)
|
||||||
|
rows = list(wb.active.iter_rows(values_only=True))
|
||||||
|
wb.close()
|
||||||
|
# rows[3] = header; rows[4] = numeric column index; rows[5] = 'Итого'; rows[6+] = data
|
||||||
|
seen = {}
|
||||||
|
for r in rows[6:]:
|
||||||
|
name = r[0]
|
||||||
|
if not name or not isinstance(name, str) or name.strip() == '': continue
|
||||||
|
# Deduplicate: keep entry with highest area (xlsx может содержать дубли названий)
|
||||||
|
key = name.strip()
|
||||||
|
prev = seen.get(key)
|
||||||
|
cur_area = r[1] if r[1] is not None else 0
|
||||||
|
if prev is None or (prev[1] or 0) < cur_area:
|
||||||
|
seen[key] = r
|
||||||
|
parts = [f"('{SNAP_APR26}', {esc(k)}, {num(r[1])}, {integer(r[2])}, "
|
||||||
|
f"{integer(r[3])}, {integer(r[4])}, {num(r[5])})"
|
||||||
|
for k, r in seen.items()]
|
||||||
|
BATCH = 500
|
||||||
|
for i in range(0, len(parts), BATCH):
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_developers_full (snapshot_date, developer_name, area_thousand_sqm,
|
||||||
|
permits_count, houses_count, flats_count, market_share_pct)
|
||||||
|
VALUES {','.join(parts[i:i+BATCH])}
|
||||||
|
ON CONFLICT (snapshot_date, developer_name) DO UPDATE SET
|
||||||
|
area_thousand_sqm = EXCLUDED.area_thousand_sqm,
|
||||||
|
permits_count = EXCLUDED.permits_count,
|
||||||
|
houses_count = EXCLUDED.houses_count,
|
||||||
|
flats_count = EXCLUDED.flats_count,
|
||||||
|
market_share_pct = EXCLUDED.market_share_pct;
|
||||||
|
""")
|
||||||
|
print(f'developers_full: {len(parts)} (deduped from {len(rows)-6})')
|
||||||
|
|
||||||
|
# ─────────── 2. escrow_rf.xlsx → domrf_escrow_banks ───────────
|
||||||
|
wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'escrow_rf_26-04-2026.xlsx'),
|
||||||
|
read_only=True, data_only=True)
|
||||||
|
rows = list(wb.active.iter_rows(values_only=True))
|
||||||
|
wb.close()
|
||||||
|
parts = []
|
||||||
|
for r in rows[6:]:
|
||||||
|
name = r[0]
|
||||||
|
if not name or not isinstance(name, str) or name.strip() == '' or name == 'Итого': continue
|
||||||
|
parts.append(f"('{SNAP_APR26}', {esc(name.strip())}, {integer(r[1])}, {integer(r[2])}, "
|
||||||
|
f"{num(r[3])}, {integer(r[4])}, {integer(r[5])})")
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_escrow_banks (snapshot_date, bank_name, houses_count, permits_count,
|
||||||
|
area_thousand_sqm, developers_count, regions_count)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, bank_name) DO UPDATE SET
|
||||||
|
houses_count = EXCLUDED.houses_count,
|
||||||
|
permits_count = EXCLUDED.permits_count,
|
||||||
|
area_thousand_sqm = EXCLUDED.area_thousand_sqm,
|
||||||
|
developers_count = EXCLUDED.developers_count,
|
||||||
|
regions_count = EXCLUDED.regions_count;
|
||||||
|
""")
|
||||||
|
print(f'escrow_banks: {len(parts)}')
|
||||||
|
|
||||||
|
# ─────────── 3. guaranty.xlsx → domrf_guaranty_regions ───────────
|
||||||
|
wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'guaranty_26-04-2026.xlsx'),
|
||||||
|
read_only=True, data_only=True)
|
||||||
|
rows = list(wb.active.iter_rows(values_only=True))
|
||||||
|
wb.close()
|
||||||
|
# rows[6] = РФ; rows[7..] = ФО + регионы
|
||||||
|
FO_NAMES = {'Центральный ФО', 'Северо-Западный ФО', 'Южный ФО', 'Северо-Кавказский ФО',
|
||||||
|
'Приволжский ФО', 'Уральский ФО', 'Сибирский ФО', 'Дальневосточный ФО',
|
||||||
|
'Новые субъекты Российской Федерации'}
|
||||||
|
parts = []
|
||||||
|
for r in rows[6:]:
|
||||||
|
name = r[0]
|
||||||
|
if not name or not isinstance(name, str) or name.strip() == '': continue
|
||||||
|
name = name.strip()
|
||||||
|
if name == 'Российская Федерация': level = 'rf'
|
||||||
|
elif name in FO_NAMES: level = 'fo'
|
||||||
|
else: level = 'region'
|
||||||
|
parts.append(f"('{SNAP_APR26}', {esc(name)}, '{level}', "
|
||||||
|
f"{integer(r[1])}, {num(r[2])}, {integer(r[3])}, "
|
||||||
|
f"{integer(r[4])}, {num(r[5])}, {integer(r[6])}, {num(r[7])}, "
|
||||||
|
f"{integer(r[8])}, {num(r[9])}, {integer(r[10])}, "
|
||||||
|
f"{integer(r[11])}, {num(r[12])}, {integer(r[13])})")
|
||||||
|
BATCH = 100
|
||||||
|
for i in range(0, len(parts), BATCH):
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_guaranty_regions (snapshot_date, territory_name, territory_level,
|
||||||
|
total_permits, total_area_th_sqm, total_developers,
|
||||||
|
fz214_permits, fz214_area_th_sqm, fz214_developers, fz214_pct_of_total,
|
||||||
|
escrow_permits, escrow_area_th_sqm, escrow_developers,
|
||||||
|
pp480_permits, pp480_area_th_sqm, pp480_developers)
|
||||||
|
VALUES {','.join(parts[i:i+BATCH])}
|
||||||
|
ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
|
||||||
|
total_permits = EXCLUDED.total_permits,
|
||||||
|
total_area_th_sqm = EXCLUDED.total_area_th_sqm,
|
||||||
|
total_developers = EXCLUDED.total_developers,
|
||||||
|
fz214_permits = EXCLUDED.fz214_permits,
|
||||||
|
fz214_area_th_sqm = EXCLUDED.fz214_area_th_sqm,
|
||||||
|
fz214_developers = EXCLUDED.fz214_developers,
|
||||||
|
fz214_pct_of_total = EXCLUDED.fz214_pct_of_total,
|
||||||
|
escrow_permits = EXCLUDED.escrow_permits,
|
||||||
|
escrow_area_th_sqm = EXCLUDED.escrow_area_th_sqm,
|
||||||
|
escrow_developers = EXCLUDED.escrow_developers,
|
||||||
|
pp480_permits = EXCLUDED.pp480_permits,
|
||||||
|
pp480_area_th_sqm = EXCLUDED.pp480_area_th_sqm,
|
||||||
|
pp480_developers = EXCLUDED.pp480_developers;
|
||||||
|
""")
|
||||||
|
print(f'guaranty_regions: {len(parts)}')
|
||||||
|
|
||||||
|
# ─────────── 4. housing_info.pdf → domrf_housing_summary + planned_commissioning ───────────
|
||||||
|
# extracted manually from PDF
|
||||||
|
housing_summary = [
|
||||||
|
# (territory, level, mechanism, devs, permits, decls, houses, area, flats_th)
|
||||||
|
('Российская Федерация', 'rf', 'all', 4336, 7257, 7447, 11510, 119234, 2424),
|
||||||
|
('Российская Федерация', 'rf', 'escrow', 4285, 7170, 7359, 11385, 117670, 2394),
|
||||||
|
('Российская Федерация', 'rf', 'fund', 42, 61, 62, 85, 1104, 22),
|
||||||
|
('Российская Федерация', 'rf', 'no_attraction', 15, 28, 28, 40, 460, 8),
|
||||||
|
]
|
||||||
|
# top-10 regions (escrow / fund / no_attraction / total) area_th_sqm
|
||||||
|
region_breakdown = [
|
||||||
|
# (region, escrow, fund, no_attr, total)
|
||||||
|
('Город Москва', 15670, 364, 286, 16320),
|
||||||
|
('Краснодарский край', 8179, 24, 4, 8206),
|
||||||
|
('Московская область', 7905, 81, None, 7986),
|
||||||
|
('Свердловская область', 5637, 17, None, 5654),
|
||||||
|
('Город Санкт-Петербург', 5258, None, None, 5258),
|
||||||
|
('Ростовская область', 4926, None, None, 4926),
|
||||||
|
('Ленинградская область', 4640, 60, None, 4700),
|
||||||
|
('Новосибирская область', 3580, 107, None, 3687),
|
||||||
|
('Республика Башкортостан',3583, None, None, 3583),
|
||||||
|
('Тюменская область', 3513, None, 11, 3524),
|
||||||
|
]
|
||||||
|
city_breakdown = [
|
||||||
|
# (city, escrow, fund, no_attr, total)
|
||||||
|
('Москва', 15670, 364, 286, 16320),
|
||||||
|
('Краснодар', 5791, None, None, 5791),
|
||||||
|
('Екатеринбург', 5294, 17, None, 5310),
|
||||||
|
('Санкт-Петербург', 5258, None, None, 5258),
|
||||||
|
('Ростов-на-Дону', 3605, None, None, 3605),
|
||||||
|
('Тюмень', 3204, None, 11, 3215),
|
||||||
|
('Новосибирск', 2755, 104, None, 2859),
|
||||||
|
('Уфа', 2786, None, None, 2786),
|
||||||
|
('Казань', 2205, None, None, 2205),
|
||||||
|
('Владивосток', 2134, None, None, 2134),
|
||||||
|
]
|
||||||
|
parts = []
|
||||||
|
for (terr, level, mech, devs, permits, decls, houses, area, flats) in housing_summary:
|
||||||
|
parts.append(f"('{SNAP_APR26}', {esc(terr)}, '{level}', '{mech}', {integer(devs)}, "
|
||||||
|
f"{integer(permits)}, {integer(decls)}, {integer(houses)}, {num(area)}, {num(flats)})")
|
||||||
|
for breakdown, level in [(region_breakdown, 'region'), (city_breakdown, 'city')]:
|
||||||
|
for (terr, escrow, fund, no_attr, total) in breakdown:
|
||||||
|
for mech, area in [('escrow', escrow), ('fund', fund),
|
||||||
|
('no_attraction', no_attr), ('all', total)]:
|
||||||
|
if area is None: continue
|
||||||
|
parts.append(f"('{SNAP_APR26}', {esc(terr)}, '{level}', '{mech}', NULL, "
|
||||||
|
f"NULL, NULL, NULL, {num(area)}, NULL)")
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_housing_summary (snapshot_date, territory_name, territory_level,
|
||||||
|
mechanism, developers_count, permits_count, declarations_count, houses_count,
|
||||||
|
area_thousand_sqm, flats_thousand)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, territory_name, mechanism) DO UPDATE SET
|
||||||
|
developers_count = EXCLUDED.developers_count,
|
||||||
|
permits_count = EXCLUDED.permits_count,
|
||||||
|
declarations_count = EXCLUDED.declarations_count,
|
||||||
|
houses_count = EXCLUDED.houses_count,
|
||||||
|
area_thousand_sqm = EXCLUDED.area_thousand_sqm,
|
||||||
|
flats_thousand = EXCLUDED.flats_thousand;
|
||||||
|
""")
|
||||||
|
print(f'housing_summary: {len(parts)}')
|
||||||
|
|
||||||
|
# planned_commissioning (year, escrow, fund, no_attr, total)
|
||||||
|
pc = [
|
||||||
|
(2026, 37084, 685, 154, 37923),
|
||||||
|
(2027, 34896, 246, 221, 35363),
|
||||||
|
(2028, 26360, 113, 30, 26503),
|
||||||
|
(2029, 10924, 19, 24, 10967),
|
||||||
|
(2030, 4245, 13, 32, 4290),
|
||||||
|
(2031, 1797, 28, None, 1825),
|
||||||
|
(2032, 1323, None, None, 1323),
|
||||||
|
(2033, 351, None, None, 351),
|
||||||
|
(2034, 234, None, None, 234),
|
||||||
|
(2035, 67, None, None, 67),
|
||||||
|
]
|
||||||
|
parts = [f"('{SNAP_APR26}', {y}, {num(e)}, {num(f)}, {num(n)}, {num(t)})"
|
||||||
|
for (y, e, f, n, t) in pc]
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_planned_commissioning (snapshot_date, plan_year, escrow_th_sqm,
|
||||||
|
fund_th_sqm, no_attraction_th_sqm, total_th_sqm)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, plan_year) DO UPDATE SET
|
||||||
|
escrow_th_sqm = EXCLUDED.escrow_th_sqm,
|
||||||
|
fund_th_sqm = EXCLUDED.fund_th_sqm,
|
||||||
|
no_attraction_th_sqm = EXCLUDED.no_attraction_th_sqm,
|
||||||
|
total_th_sqm = EXCLUDED.total_th_sqm;
|
||||||
|
""")
|
||||||
|
print(f'planned_commissioning: {len(parts)}')
|
||||||
|
|
||||||
|
# ─────────── 5. sold_out_info.pdf → 3 tables ───────────
|
||||||
|
# RF total
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_sold_out (snapshot_date, territory_name, total_area_th_sqm,
|
||||||
|
sold_area_th_sqm, unsold_area_th_sqm, not_open_area_th_sqm,
|
||||||
|
sold_pct, unsold_pct, not_open_pct, avg_price_per_sqm, attracted_funds_mln_rub)
|
||||||
|
VALUES ('{SNAP_SOLD}', 'Российская Федерация',
|
||||||
|
118876, 36749, 53044, 29084, 31, 45, 24, 216316, 7949316)
|
||||||
|
ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
|
||||||
|
total_area_th_sqm = EXCLUDED.total_area_th_sqm,
|
||||||
|
sold_area_th_sqm = EXCLUDED.sold_area_th_sqm,
|
||||||
|
unsold_area_th_sqm = EXCLUDED.unsold_area_th_sqm,
|
||||||
|
not_open_area_th_sqm = EXCLUDED.not_open_area_th_sqm,
|
||||||
|
sold_pct = EXCLUDED.sold_pct,
|
||||||
|
unsold_pct = EXCLUDED.unsold_pct,
|
||||||
|
not_open_pct = EXCLUDED.not_open_pct,
|
||||||
|
avg_price_per_sqm = EXCLUDED.avg_price_per_sqm,
|
||||||
|
attracted_funds_mln_rub = EXCLUDED.attracted_funds_mln_rub;
|
||||||
|
""")
|
||||||
|
# Top-8 regions: (region, total, sold%, unsold%, not_open%)
|
||||||
|
so_regions = [
|
||||||
|
('Город Москва', 16405, 47, 44, None),
|
||||||
|
('Краснодарский край', 8286, 20, 39, 41),
|
||||||
|
('Московская область', 7942, 37, 43, None),
|
||||||
|
('Свердловская область', 5686, 29, 54, None),
|
||||||
|
('Город Санкт-Петербург', 5336, 40, 44, None),
|
||||||
|
('Ростовская область', 4990, None, 48, None),
|
||||||
|
('Ленинградская область', 4705, None, 44, None),
|
||||||
|
('Тюменская область', 3573, None, 50, None),
|
||||||
|
]
|
||||||
|
parts = []
|
||||||
|
for (r, total, sp, up, nop) in so_regions:
|
||||||
|
parts.append(f"('{SNAP_SOLD}', {esc(r)}, {num(total)}, NULL, NULL, NULL, "
|
||||||
|
f"{num(sp)}, {num(up)}, {num(nop)}, NULL, NULL)")
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_sold_out (snapshot_date, territory_name, total_area_th_sqm,
|
||||||
|
sold_area_th_sqm, unsold_area_th_sqm, not_open_area_th_sqm,
|
||||||
|
sold_pct, unsold_pct, not_open_pct, avg_price_per_sqm, attracted_funds_mln_rub)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
|
||||||
|
total_area_th_sqm = EXCLUDED.total_area_th_sqm,
|
||||||
|
sold_pct = EXCLUDED.sold_pct,
|
||||||
|
unsold_pct = EXCLUDED.unsold_pct,
|
||||||
|
not_open_pct = EXCLUDED.not_open_pct;
|
||||||
|
""")
|
||||||
|
print(f'sold_out: 1 RF + {len(parts)} regions')
|
||||||
|
|
||||||
|
# by_year — (year, total_th_sqm, pct, sold%, unsold%, not_open%)
|
||||||
|
so_year = [
|
||||||
|
('2026', 40201, 34, 52, 38, 10),
|
||||||
|
('2027', 34880, 29, 30, 53, 17),
|
||||||
|
('2028', 25363, 21, 17, 52, 31),
|
||||||
|
('2029', 10359, 9, None, 37, 54),
|
||||||
|
('2030', 4127, 3, None, None, 71),
|
||||||
|
('2031+', 3946, 3, None, None, 65),
|
||||||
|
]
|
||||||
|
parts = [f"('{SNAP_SOLD}', '{y}', {num(t)}, {num(p)}, {num(s)}, {num(u)}, {num(n)})"
|
||||||
|
for (y, t, p, s, u, n) in so_year]
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_sold_out_by_year (snapshot_date, plan_year, total_th_sqm,
|
||||||
|
pct_of_total, sold_pct, unsold_pct, not_open_pct)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, plan_year) DO UPDATE SET
|
||||||
|
total_th_sqm = EXCLUDED.total_th_sqm,
|
||||||
|
pct_of_total = EXCLUDED.pct_of_total,
|
||||||
|
sold_pct = EXCLUDED.sold_pct,
|
||||||
|
unsold_pct = EXCLUDED.unsold_pct,
|
||||||
|
not_open_pct = EXCLUDED.not_open_pct;
|
||||||
|
""")
|
||||||
|
print(f'sold_out_by_year: {len(parts)}')
|
||||||
|
|
||||||
|
# by_progress
|
||||||
|
so_prog = [
|
||||||
|
('not_open', 29084, 24, None, 100),
|
||||||
|
('0-20', 26170, 22, 9, 91),
|
||||||
|
('21-40', 21790, 18, 30, 70),
|
||||||
|
('41-60', 17409, 15, 50, 50),
|
||||||
|
('61-80', 13402, 11, 69, 31),
|
||||||
|
('80+', 11021, 9, 90, 9),
|
||||||
|
]
|
||||||
|
parts = [f"('{SNAP_SOLD}', '{b}', {num(t)}, {num(p)}, {num(s)}, {num(u)})"
|
||||||
|
for (b, t, p, s, u) in so_prog]
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_sold_out_by_progress (snapshot_date, progress_bucket, total_th_sqm,
|
||||||
|
pct_of_total, sold_pct_within, unsold_pct_within)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, progress_bucket) DO UPDATE SET
|
||||||
|
total_th_sqm = EXCLUDED.total_th_sqm,
|
||||||
|
pct_of_total = EXCLUDED.pct_of_total,
|
||||||
|
sold_pct_within = EXCLUDED.sold_pct_within,
|
||||||
|
unsold_pct_within = EXCLUDED.unsold_pct_within;
|
||||||
|
""")
|
||||||
|
print(f'sold_out_by_progress: {len(parts)}')
|
||||||
|
|
||||||
|
# ─────────── 6. mortgage_rates.pdf → domrf_mortgage_rates ───────────
|
||||||
|
rates = [
|
||||||
|
# (bank, primary, secondary, refi, note)
|
||||||
|
('Сбербанк', 20.2, 19.7, 21.5, None),
|
||||||
|
('ВТБ', 19.9, 19.9, 19.6, None),
|
||||||
|
('Банк ДОМ.РФ', 18.2, 18.2, 22.1, None),
|
||||||
|
('Альфа-банк', 19.49, 19.49, 18.99, None),
|
||||||
|
('Совкомбанк', 19.99, 20.49, 19.99, None),
|
||||||
|
('Т-Банк', 16.9, 16.9, 16.9, None),
|
||||||
|
('Банк Санкт-Петербург', 18.49, 18.49, 18.49, None),
|
||||||
|
('Промсвязьбанк', 19.49, 18.79, 18.99, None),
|
||||||
|
('Московский Кредитный Банк', 18.6, 18.6, 18.6, None),
|
||||||
|
('Уралсиб', 17.89, 17.89, 18.99, None),
|
||||||
|
('Абсолют банк', 19.35, 19.35, 19.35, None),
|
||||||
|
('Металлинвестбанк', 18.1, 18.1, 18.5, 'к пред.неделе -0.7 пп'),
|
||||||
|
('Газпромбанк', None, None, None, 'Максимальный срок – 19 лет, рефинанс не предоставляет'),
|
||||||
|
('Азиатско-Тихоокеанский Банк', 18.9, 18.9, 18.9, None),
|
||||||
|
('Россельхозбанк', None, None, None, 'не предоставляет'),
|
||||||
|
('Банк Кубань Кредит', 16.4, 16.4, 15.9, None),
|
||||||
|
('Транскапиталбанк', 19.6, 19.6, 19.6, None),
|
||||||
|
('УБРиР', None, None, None, 'не предоставляет'),
|
||||||
|
('Ак Барс Банк', None, None, None, 'ПВ от 50%, рефинанс не предоставляет'),
|
||||||
|
('Средневзвешенная', 19.90, 19.59, 18.81, 'aggregate top-20'),
|
||||||
|
]
|
||||||
|
parts = [f"('{SNAP_MORTGAGE}', {esc(b)}, {num(p)}, {num(s)}, {num(r)}, {esc(n)})"
|
||||||
|
for (b, p, s, r, n) in rates]
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_mortgage_rates (snapshot_date, bank_name, primary_rate, secondary_rate, refinance_rate, note)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, bank_name) DO UPDATE SET
|
||||||
|
primary_rate = EXCLUDED.primary_rate,
|
||||||
|
secondary_rate = EXCLUDED.secondary_rate,
|
||||||
|
refinance_rate = EXCLUDED.refinance_rate,
|
||||||
|
note = EXCLUDED.note;
|
||||||
|
""")
|
||||||
|
print(f'mortgage_rates: {len(parts)}')
|
||||||
|
|
||||||
|
# Summary
|
||||||
|
print('\n--- summary ---')
|
||||||
|
out = psql("""
|
||||||
|
SELECT 'developers_full' tab, COUNT(*) FROM domrf_developers_full
|
||||||
|
UNION ALL SELECT 'escrow_banks', COUNT(*) FROM domrf_escrow_banks
|
||||||
|
UNION ALL SELECT 'guaranty_regions', COUNT(*) FROM domrf_guaranty_regions
|
||||||
|
UNION ALL SELECT 'housing_summary', COUNT(*) FROM domrf_housing_summary
|
||||||
|
UNION ALL SELECT 'planned_commissioning', COUNT(*) FROM domrf_planned_commissioning
|
||||||
|
UNION ALL SELECT 'sold_out', COUNT(*) FROM domrf_sold_out
|
||||||
|
UNION ALL SELECT 'sold_out_by_year', COUNT(*) FROM domrf_sold_out_by_year
|
||||||
|
UNION ALL SELECT 'sold_out_by_progress', COUNT(*) FROM domrf_sold_out_by_progress
|
||||||
|
UNION ALL SELECT 'mortgage_rates', COUNT(*) FROM domrf_mortgage_rates
|
||||||
|
ORDER BY 1;
|
||||||
|
""")
|
||||||
|
print(out)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
181
data/sql/30_scrape_domrf.py
Normal file
181
data/sql/30_scrape_domrf.py
Normal file
|
|
@ -0,0 +1,181 @@
|
||||||
|
"""DOM.RF analytics scraper using Playwright (headless Chromium).
|
||||||
|
|
||||||
|
Bypasses:
|
||||||
|
- ServicePipe WAF (Chromium executes JS challenge automatically)
|
||||||
|
- Lazy-loading (wait for `networkidle` + page-specific triggers)
|
||||||
|
- Cookie session (Chromium maintains it across navigations)
|
||||||
|
|
||||||
|
Captures: every JSON response from /api/* endpoints on the same domain.
|
||||||
|
Saves: data/raw/domrf_full/<section>/<endpoint>.json (one JSON per endpoint).
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
PYTHONIOENCODING=utf-8 python data/sql/30_scrape_domrf.py [--headed] [--page <name>]
|
||||||
|
|
||||||
|
Run --headed first to verify WAF passes; switch to default headless after.
|
||||||
|
"""
|
||||||
|
import asyncio, json, os, re, sys
|
||||||
|
from urllib.parse import urlsplit, parse_qs, urlencode, urlunsplit
|
||||||
|
from playwright.async_api import async_playwright
|
||||||
|
|
||||||
|
ROOT = os.path.join(os.path.dirname(__file__), '..', 'raw', 'domrf_full')
|
||||||
|
BASE = 'https://xn--80az8a.xn--d1aqf.xn--p1ai'
|
||||||
|
|
||||||
|
# 12 analytics pages — name (used as folder) → URL path
|
||||||
|
PAGES = [
|
||||||
|
('housing', '/аналитика/показатели_жилищного_строительства'),
|
||||||
|
('launch', '/аналитика/запуски-и-вводы?repYear=2026&calculationType=SQUARE&period=YEAR'),
|
||||||
|
('share_construction','/аналитика/долевое_строительство'),
|
||||||
|
('housing_dev', '/аналитика/жилищное_строительство'),
|
||||||
|
('escrow', '/аналитика/эскроу'),
|
||||||
|
('realization', '/аналитика/реализация_строящихся_квартир'),
|
||||||
|
('sold_ready', '/аналитика/распроданность-стройготовность?repYear=2026&repMonth=3'),
|
||||||
|
('quartirografia', '/аналитика/квартирография'),
|
||||||
|
('commissioning', '/аналитика/ввод_жилья'),
|
||||||
|
('mortgage_rates', '/аналитика/ставки_предложений_по_ипотеке'),
|
||||||
|
('mortgage_stats', '/аналитика/ипотечное_кредитование'),
|
||||||
|
('stat_series', '/аналитика/статистические_ряды'),
|
||||||
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def safe_filename(url: str) -> str:
|
||||||
|
"""Turn API URL into safe filename: path + query → snake_case."""
|
||||||
|
parts = urlsplit(url)
|
||||||
|
base = parts.path.split('/api/')[-1].strip('/')
|
||||||
|
base = re.sub(r'[^A-Za-z0-9_\-]', '_', base)[:100]
|
||||||
|
if parts.query:
|
||||||
|
q = re.sub(r'[^A-Za-z0-9_\-]', '_', parts.query)[:80]
|
||||||
|
base = f'{base}__{q}'
|
||||||
|
return base + '.json'
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade_size(url: str) -> str:
|
||||||
|
"""If URL has size=15 (or any small value), upgrade to size=1000 to get full data."""
|
||||||
|
parts = urlsplit(url)
|
||||||
|
if not parts.query:
|
||||||
|
return url
|
||||||
|
qs = parse_qs(parts.query, keep_blank_values=True)
|
||||||
|
if 'size' in qs:
|
||||||
|
qs['size'] = ['1000']
|
||||||
|
new_q = urlencode({k: v[0] for k, v in qs.items()})
|
||||||
|
return urlunsplit((parts.scheme, parts.netloc, parts.path, new_q, parts.fragment))
|
||||||
|
|
||||||
|
|
||||||
|
async def scrape_page(browser, name, path):
|
||||||
|
out_dir = os.path.join(ROOT, name)
|
||||||
|
os.makedirs(out_dir, exist_ok=True)
|
||||||
|
print(f'\n=== {name} ({path}) ===')
|
||||||
|
context = await browser.new_context(
|
||||||
|
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||||
|
'(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36',
|
||||||
|
locale='ru-RU',
|
||||||
|
viewport={'width': 1920, 'height': 1080},
|
||||||
|
)
|
||||||
|
page = await context.new_page()
|
||||||
|
|
||||||
|
seen_urls = set()
|
||||||
|
saved = []
|
||||||
|
|
||||||
|
async def on_response(resp):
|
||||||
|
url = resp.url
|
||||||
|
# Match same-origin /api/ endpoints, skip yandex/sendsay trackers
|
||||||
|
if 'xn--80az8a.xn--d1aqf.xn--p1ai' not in url or '/api/' not in url:
|
||||||
|
return
|
||||||
|
ct = (resp.headers.get('content-type') or '').lower()
|
||||||
|
if 'json' not in ct:
|
||||||
|
return
|
||||||
|
if url in seen_urls:
|
||||||
|
return
|
||||||
|
seen_urls.add(url)
|
||||||
|
try:
|
||||||
|
text = await resp.text()
|
||||||
|
fname = safe_filename(url)
|
||||||
|
with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
|
||||||
|
f.write(text)
|
||||||
|
saved.append((fname, len(text), url))
|
||||||
|
print(f' saved {fname} ({len(text)}b)')
|
||||||
|
except Exception as e:
|
||||||
|
print(f' ERR {url}: {e}')
|
||||||
|
|
||||||
|
page.on('response', lambda r: asyncio.create_task(on_response(r)))
|
||||||
|
|
||||||
|
full_url = BASE + path
|
||||||
|
try:
|
||||||
|
await page.goto(full_url, wait_until='domcontentloaded', timeout=60_000)
|
||||||
|
# WAF challenge usually completes within 1-3 seconds
|
||||||
|
try:
|
||||||
|
await page.wait_for_load_state('networkidle', timeout=30_000)
|
||||||
|
except:
|
||||||
|
pass
|
||||||
|
# Extra wait for lazy-loaded API calls
|
||||||
|
await page.wait_for_timeout(8000)
|
||||||
|
# Scroll multiple times to trigger any IntersectionObserver-bound fetches
|
||||||
|
for _ in range(3):
|
||||||
|
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
|
||||||
|
await page.wait_for_timeout(1500)
|
||||||
|
await page.evaluate('window.scrollTo(0, document.body.scrollHeight / 2)')
|
||||||
|
await page.wait_for_timeout(1500)
|
||||||
|
await page.evaluate('window.scrollTo(0, 0)')
|
||||||
|
await page.wait_for_timeout(1500)
|
||||||
|
# Click first interactive button (triggers React state → API)
|
||||||
|
try:
|
||||||
|
await page.evaluate('''() => {
|
||||||
|
const btn = document.querySelector('button:not([disabled])');
|
||||||
|
if (btn) btn.click();
|
||||||
|
}''')
|
||||||
|
await page.wait_for_timeout(3000)
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
|
||||||
|
# Re-fetch with size=1000 for endpoints that have small page-size
|
||||||
|
for url in list(seen_urls):
|
||||||
|
up = upgrade_size(url)
|
||||||
|
if up != url and up not in seen_urls:
|
||||||
|
try:
|
||||||
|
resp = await page.evaluate(
|
||||||
|
'''(u) => fetch(u, {credentials:'include'})
|
||||||
|
.then(r => r.text()).then(t => ({status: 200, text: t}))''',
|
||||||
|
up,
|
||||||
|
)
|
||||||
|
if resp and resp.get('text'):
|
||||||
|
seen_urls.add(up)
|
||||||
|
fname = safe_filename(up)
|
||||||
|
with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
|
||||||
|
f.write(resp['text'])
|
||||||
|
saved.append((fname, len(resp['text']), up))
|
||||||
|
print(f' upgraded {fname} ({len(resp["text"])}b)')
|
||||||
|
except Exception as e:
|
||||||
|
print(f' upgrade err {up}: {e}')
|
||||||
|
finally:
|
||||||
|
await context.close()
|
||||||
|
return saved
|
||||||
|
|
||||||
|
|
||||||
|
async def main():
|
||||||
|
headed = '--headed' in sys.argv
|
||||||
|
target = None
|
||||||
|
if '--page' in sys.argv:
|
||||||
|
target = sys.argv[sys.argv.index('--page') + 1]
|
||||||
|
|
||||||
|
pages = [(n, p) for n, p in PAGES if target is None or n == target]
|
||||||
|
print(f'Scraping {len(pages)} pages, headed={headed}')
|
||||||
|
os.makedirs(ROOT, exist_ok=True)
|
||||||
|
|
||||||
|
async with async_playwright() as pw:
|
||||||
|
browser = await pw.chromium.launch(headless=not headed)
|
||||||
|
all_saved = {}
|
||||||
|
for name, path in pages:
|
||||||
|
try:
|
||||||
|
all_saved[name] = await scrape_page(browser, name, path)
|
||||||
|
except Exception as e:
|
||||||
|
print(f'PAGE {name} FAILED: {e}')
|
||||||
|
all_saved[name] = []
|
||||||
|
await browser.close()
|
||||||
|
|
||||||
|
print('\n=== SUMMARY ===')
|
||||||
|
for name, saved in all_saved.items():
|
||||||
|
total = sum(s[1] for s in saved)
|
||||||
|
print(f' {name:22s} files={len(saved):3d} total={total:>10}b')
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
asyncio.run(main())
|
||||||
166
data/sql/31_schema_domrf_full.sql
Normal file
166
data/sql/31_schema_domrf_full.sql
Normal file
|
|
@ -0,0 +1,166 @@
|
||||||
|
-- DOM.RF analytics, normalized schema for scraper output (data/raw/domrf_full/*).
|
||||||
|
-- Snapshot date stamped per scrape; idempotent UPSERT loaders.
|
||||||
|
|
||||||
|
-- ── 1. Launch (запуски + ввод в эксплуатацию) ────────────────────────────────
|
||||||
|
-- Topology: page returns top-N entities (developers/regions/areas/fo) with launch (app)
|
||||||
|
-- and commissioning (rnv) values. Single table covers all 4 dimensions via dim_type.
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_launch_top (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
rep_year INT NOT NULL,
|
||||||
|
calc_type TEXT NOT NULL, -- 'SQUARE' (м²) | 'HOUSES' (шт) | 'FLATS' (шт)
|
||||||
|
dim_type TEXT NOT NULL, -- 'developer' | 'region' | 'area' | 'fo'
|
||||||
|
metric_type TEXT NOT NULL, -- 'app' (запуски) | 'rnv' (ввод)
|
||||||
|
entity_id TEXT NOT NULL, -- '6072_0' (developer) | '50' (region) | 'Москва' (area)
|
||||||
|
entity_name TEXT NOT NULL,
|
||||||
|
value NUMERIC NOT NULL,
|
||||||
|
PRIMARY KEY (snapshot_date, rep_year, calc_type, dim_type, metric_type, entity_id)
|
||||||
|
);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_launch_top_dim ON domrf_launch_top (dim_type, metric_type, value DESC);
|
||||||
|
|
||||||
|
-- Объёмы по классам жилья
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_launch_obj_class (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
rep_year INT NOT NULL,
|
||||||
|
calc_type TEXT NOT NULL,
|
||||||
|
obj_class_cd INT NOT NULL, -- 1=Типовой, 2=Комфорт, 3=Бизнес, 4=Элитный
|
||||||
|
obj_class_desc TEXT NOT NULL,
|
||||||
|
app_value NUMERIC,
|
||||||
|
rnv_value NUMERIC,
|
||||||
|
PRIMARY KEY (snapshot_date, rep_year, calc_type, obj_class_cd)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Time series: год × месяц → запуски/вводы
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_launch_monthly (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
calc_type TEXT NOT NULL,
|
||||||
|
rep_year INT NOT NULL,
|
||||||
|
rep_month INT NOT NULL, -- 1..12
|
||||||
|
app_value NUMERIC,
|
||||||
|
rnv_value NUMERIC,
|
||||||
|
PRIMARY KEY (snapshot_date, calc_type, rep_year, rep_month)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ── 2. Реализация / распроданность-стройготовность (sold_ready) ──────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_sold_ready_index (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
rep_year INT NOT NULL,
|
||||||
|
rep_month INT NOT NULL,
|
||||||
|
square_sum NUMERIC, -- всего стр. площади РФ, м²
|
||||||
|
sold_perc NUMERIC, -- % проданного
|
||||||
|
sold_sum NUMERIC, -- проданная площадь, м²
|
||||||
|
sold_ready_perc NUMERIC, -- "распроданность ÷ готовность"
|
||||||
|
ready_perc NUMERIC, -- % готовности
|
||||||
|
PRIMARY KEY (snapshot_date, rep_year, rep_month)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Reality breakdowns: foChart / regionChart / cityChart / devChart …
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_sold_ready_breakdown (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
rep_year INT NOT NULL,
|
||||||
|
rep_month INT NOT NULL,
|
||||||
|
chart_type TEXT NOT NULL, -- 'foChart' | 'regionChart' | 'cityChart' | 'devChart' | ...
|
||||||
|
entity_key TEXT NOT NULL, -- название/код
|
||||||
|
square_sum NUMERIC,
|
||||||
|
sold_perc NUMERIC,
|
||||||
|
ready_perc NUMERIC,
|
||||||
|
sold_ready_perc NUMERIC,
|
||||||
|
fo_cd INT,
|
||||||
|
extra JSONB, -- backup для прочих полей
|
||||||
|
PRIMARY KEY (snapshot_date, rep_year, rep_month, chart_type, entity_key)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- Динамика: chartType × год × месяц → значение
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_sold_ready_dynamics (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
dynamic_chart_type TEXT NOT NULL, -- 'squareSumChart' | 'soldPercChart' | 'readyPercChart' | 'soldReadyPercChart'
|
||||||
|
rep_year INT NOT NULL,
|
||||||
|
rep_month INT NOT NULL,
|
||||||
|
value NUMERIC,
|
||||||
|
PRIMARY KEY (snapshot_date, dynamic_chart_type, rep_year, rep_month)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ── 3. Share construction (долевое стр-во / project finance) ────────────────
|
||||||
|
-- /аналитика/api/project/finance/dashboard returns rows per region/FO/RF.
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_project_finance (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
report_date DATE,
|
||||||
|
subject_type TEXT, -- 'rf' | 'fo' | 'region'
|
||||||
|
subject_name TEXT,
|
||||||
|
region_cd INT,
|
||||||
|
fo_cd INT,
|
||||||
|
fo_desc TEXT,
|
||||||
|
rns_cnt INT, -- разрешения на стр-во
|
||||||
|
dev_cnt INT, -- застройщики
|
||||||
|
liv_sq_amt NUMERIC, -- жилая площадь м²
|
||||||
|
guaranty_escrow_rns_cnt INT, -- эскроу разрешения
|
||||||
|
guaranty_escrow_dev_cnt INT,
|
||||||
|
guaranty_escrow_liv_sq_amt NUMERIC,
|
||||||
|
guaranty_zosg_rns_cnt INT,
|
||||||
|
guaranty_zosg_dev_cnt INT,
|
||||||
|
guaranty_zosg_liv_sq_amt NUMERIC,
|
||||||
|
guaranty_rns_cnt INT, -- всего по 214-ФЗ
|
||||||
|
guaranty_dev_cnt INT,
|
||||||
|
guaranty_liv_sq_amt NUMERIC,
|
||||||
|
nonguaranty_rns_cnt INT, -- ПП №480
|
||||||
|
nonguaranty_dev_cnt INT,
|
||||||
|
nonguaranty_liv_sq_amt NUMERIC,
|
||||||
|
extra JSONB,
|
||||||
|
id BIGSERIAL PRIMARY KEY
|
||||||
|
);
|
||||||
|
CREATE UNIQUE INDEX IF NOT EXISTS uq_project_finance_snap
|
||||||
|
ON domrf_project_finance (snapshot_date, COALESCE(subject_type, ''), COALESCE(subject_name, ''));
|
||||||
|
|
||||||
|
-- ── 4. Ввод жилья (commissioning building_summary) ──────────────────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_commissioning (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
region_id INT NOT NULL,
|
||||||
|
region_name TEXT NOT NULL,
|
||||||
|
region_type TEXT, -- 'rf' | 'fo' | 'region'
|
||||||
|
reporting_period DATE NOT NULL,
|
||||||
|
rep_year INT,
|
||||||
|
accumulated_fact_area NUMERIC,
|
||||||
|
accumulated_fact_area_multifamily NUMERIC,
|
||||||
|
accumulated_fact_area_private NUMERIC,
|
||||||
|
accumulated_fact_area_change NUMERIC,
|
||||||
|
accumulated_fact_area_change_share NUMERIC,
|
||||||
|
extra JSONB,
|
||||||
|
PRIMARY KEY (snapshot_date, region_id, reporting_period)
|
||||||
|
);
|
||||||
|
|
||||||
|
-- ── 5. Mortgage stats ───────────────────────────────────────────────────────
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_mortgage_dashboard (
|
||||||
|
snapshot_date DATE NOT NULL PRIMARY KEY,
|
||||||
|
total_credit_count INT,
|
||||||
|
total_credit_count_delta_pct NUMERIC,
|
||||||
|
primary_credit_count INT,
|
||||||
|
primary_credit_count_delta_pct NUMERIC,
|
||||||
|
secondary_credit_count INT,
|
||||||
|
secondary_credit_count_delta_pct NUMERIC,
|
||||||
|
total_credit_amount NUMERIC,
|
||||||
|
total_credit_amount_delta_pct NUMERIC,
|
||||||
|
primary_credit_amount NUMERIC,
|
||||||
|
primary_credit_amount_delta_pct NUMERIC,
|
||||||
|
secondary_credit_amount NUMERIC,
|
||||||
|
secondary_credit_amount_delta_pct NUMERIC,
|
||||||
|
total_credit_avg_rate NUMERIC,
|
||||||
|
total_credit_avg_rate_delta NUMERIC,
|
||||||
|
primary_credit_avg_rate NUMERIC,
|
||||||
|
primary_credit_avg_rate_delta NUMERIC,
|
||||||
|
secondary_credit_avg_rate NUMERIC,
|
||||||
|
secondary_credit_avg_rate_delta NUMERIC,
|
||||||
|
extra JSONB
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_mortgage_details (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
currency TEXT NOT NULL,
|
||||||
|
credit_amount_avg NUMERIC,
|
||||||
|
credit_amount_avg_delta_pct NUMERIC,
|
||||||
|
credit_avg_period NUMERIC,
|
||||||
|
credit_avg_period_delta_pct NUMERIC,
|
||||||
|
credit_debts_amount NUMERIC,
|
||||||
|
credit_debts_amount_delta_pct NUMERIC,
|
||||||
|
credit_debts_overdue_percent NUMERIC,
|
||||||
|
credit_debts_overdue_percent_delta NUMERIC,
|
||||||
|
PRIMARY KEY (snapshot_date, currency)
|
||||||
|
);
|
||||||
18
data/sql/31_schema_domrf_raw.sql
Normal file
18
data/sql/31_schema_domrf_raw.sql
Normal file
|
|
@ -0,0 +1,18 @@
|
||||||
|
-- DOM.RF analytics raw API endpoint snapshots.
|
||||||
|
-- Universal JSONB store: one row per (snapshot_date, section, endpoint).
|
||||||
|
-- Section = page name on наш.дом.рф (launch / share_construction / escrow / ...).
|
||||||
|
-- Endpoint = sanitized path+query of /api/* call.
|
||||||
|
|
||||||
|
CREATE TABLE IF NOT EXISTS domrf_raw_endpoints (
|
||||||
|
snapshot_date DATE NOT NULL,
|
||||||
|
section TEXT NOT NULL, -- 'launch' | 'sold_ready' | 'quartirografia' | ...
|
||||||
|
endpoint TEXT NOT NULL, -- 'launch_developers_app__calculationType_SQUARE_repYear_2026_size_1000'
|
||||||
|
source_url TEXT, -- inferred original URL (best effort)
|
||||||
|
payload JSONB NOT NULL,
|
||||||
|
payload_size INT, -- bytes (raw text length)
|
||||||
|
fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||||
|
PRIMARY KEY (snapshot_date, section, endpoint)
|
||||||
|
);
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_domrf_raw_section ON domrf_raw_endpoints (section);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_domrf_raw_payload_gin ON domrf_raw_endpoints USING GIN (payload);
|
||||||
378
data/sql/32_load_domrf_full.py
Normal file
378
data/sql/32_load_domrf_full.py
Normal file
|
|
@ -0,0 +1,378 @@
|
||||||
|
"""Loads scraped DOM.RF JSONs into normalized PG tables.
|
||||||
|
|
||||||
|
Reads: data/raw/domrf_full/<section>/*.json
|
||||||
|
Writes: domrf_launch_top, ..._obj_class, ..._monthly,
|
||||||
|
domrf_sold_ready_index, ..._breakdown, ..._dynamics,
|
||||||
|
domrf_project_finance, domrf_commissioning,
|
||||||
|
domrf_mortgage_dashboard, domrf_mortgage_details.
|
||||||
|
|
||||||
|
Existing tables (domrf_developers, domrf_regions, domrf_developer_aggregates,
|
||||||
|
domrf_region_aggregates, domrf_developers_full, domrf_escrow_banks,
|
||||||
|
domrf_guaranty_regions, domrf_housing_summary, domrf_sold_out,
|
||||||
|
domrf_mortgage_rates, etc.) are NOT touched.
|
||||||
|
"""
|
||||||
|
import os, json, sys, subprocess, datetime
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
ROOT = os.path.join(HERE, '..', 'raw', 'domrf_full')
|
||||||
|
PG_PORT = os.environ.get('PG_PORT', '5432')
|
||||||
|
PG_USER = os.environ.get('PG_USER', 'gendesign')
|
||||||
|
PG_DB = os.environ.get('PG_DB', 'gendesign')
|
||||||
|
PG_PASS = os.environ.get('PGPASSWORD', '')
|
||||||
|
SNAP = os.environ.get('SNAPSHOT_DATE', datetime.date.today().isoformat())
|
||||||
|
|
||||||
|
|
||||||
|
def psql(sql):
|
||||||
|
cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
|
||||||
|
'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT,
|
||||||
|
'-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet']
|
||||||
|
res = subprocess.run(cmd, input=sql, capture_output=True, text=True, encoding='utf-8')
|
||||||
|
if res.returncode != 0:
|
||||||
|
print('psql FAILED:', res.stderr[-2000:], file=sys.stderr)
|
||||||
|
raise SystemExit(res.returncode)
|
||||||
|
return res.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def esc(s):
|
||||||
|
if s is None: return 'NULL'
|
||||||
|
return "'" + str(s).replace("'", "''") + "'"
|
||||||
|
|
||||||
|
def num(v):
|
||||||
|
if v is None or v == '-' or v == '': return 'NULL'
|
||||||
|
try: return str(float(v))
|
||||||
|
except (ValueError, TypeError): return 'NULL'
|
||||||
|
|
||||||
|
def integer(v):
|
||||||
|
if v is None: return 'NULL'
|
||||||
|
try: return str(int(float(v)))
|
||||||
|
except (ValueError, TypeError): return 'NULL'
|
||||||
|
|
||||||
|
|
||||||
|
def load_json(section, fname):
|
||||||
|
p = os.path.join(ROOT, section, fname + '.json')
|
||||||
|
if not os.path.exists(p): return None
|
||||||
|
return json.load(open(p, encoding='utf-8'))
|
||||||
|
|
||||||
|
|
||||||
|
def find_files(section, prefix):
|
||||||
|
"""Find files in section/ starting with prefix (e.g. 'launch_developers_app__')."""
|
||||||
|
sec_dir = os.path.join(ROOT, section)
|
||||||
|
if not os.path.isdir(sec_dir): return []
|
||||||
|
return [f for f in os.listdir(sec_dir) if f.startswith(prefix) and f.endswith('.json')]
|
||||||
|
|
||||||
|
|
||||||
|
def best_size_file(section, prefix):
|
||||||
|
"""Pick the file with largest size= parameter (or only one)."""
|
||||||
|
files = find_files(section, prefix)
|
||||||
|
if not files: return None
|
||||||
|
# Prefer size=1000 over size=15
|
||||||
|
files.sort(key=lambda f: 0 if 'size_1000' in f else 1)
|
||||||
|
return files[0][:-5] # strip .json
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
schema = open(os.path.join(HERE, '31_schema_domrf_full.sql'), encoding='utf-8').read()
|
||||||
|
psql(schema)
|
||||||
|
print('schema applied')
|
||||||
|
|
||||||
|
# ── 1. LAUNCH ────────────────────────────────────────────────────────────
|
||||||
|
rep_year = 2026
|
||||||
|
parts = []
|
||||||
|
for dim, prefix_app, prefix_rnv in [
|
||||||
|
('developer', 'launch_developers_app__', 'launch_developers_rnv__'),
|
||||||
|
('region', 'launch_regions_app__', 'launch_regions_rnv__'),
|
||||||
|
('area', 'launch_areas_app__', 'launch_areas_rnv__'),
|
||||||
|
('fo', 'launch_federal-districts_app__', 'launch_federal-districts_rnv__'),
|
||||||
|
]:
|
||||||
|
for metric, prefix in [('app', prefix_app), ('rnv', prefix_rnv)]:
|
||||||
|
fname = best_size_file('launch', prefix)
|
||||||
|
if not fname: continue
|
||||||
|
d = load_json('launch', fname)
|
||||||
|
calc = d.get('calculationType', 'SQUARE')
|
||||||
|
for v in d.get('values', []):
|
||||||
|
eid = str(v.get('id', v.get('name', '')))
|
||||||
|
ename = v.get('name', eid)
|
||||||
|
value = v.get('value')
|
||||||
|
if value is None: continue
|
||||||
|
parts.append(f"('{SNAP}', {rep_year}, {esc(calc)}, '{dim}', '{metric}', "
|
||||||
|
f"{esc(eid)}, {esc(ename)}, {num(value)})")
|
||||||
|
if parts:
|
||||||
|
for i in range(0, len(parts), 500):
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_launch_top (snapshot_date, rep_year, calc_type, dim_type, metric_type, entity_id, entity_name, value)
|
||||||
|
VALUES {','.join(parts[i:i+500])}
|
||||||
|
ON CONFLICT (snapshot_date, rep_year, calc_type, dim_type, metric_type, entity_id) DO UPDATE SET
|
||||||
|
entity_name = EXCLUDED.entity_name, value = EXCLUDED.value;
|
||||||
|
""")
|
||||||
|
print(f'launch_top: {len(parts)}')
|
||||||
|
|
||||||
|
# obj_class
|
||||||
|
d = load_json('launch', 'launch_obj-class_table__calculationType_SQUARE_repYear_2026') \
|
||||||
|
or load_json('launch', 'launch_obj-class_table')
|
||||||
|
if d:
|
||||||
|
calc = d.get('calculationType', 'SQUARE')
|
||||||
|
parts = [f"('{SNAP}', {rep_year}, {esc(calc)}, {integer(o['objClassCd'])}, "
|
||||||
|
f"{esc(o['objClassDesc'])}, {num(o.get('appValue'))}, {num(o.get('rnvValue'))})"
|
||||||
|
for o in d.get('objClassTypes', [])]
|
||||||
|
if parts:
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_launch_obj_class (snapshot_date, rep_year, calc_type, obj_class_cd, obj_class_desc, app_value, rnv_value)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, rep_year, calc_type, obj_class_cd) DO UPDATE SET
|
||||||
|
obj_class_desc = EXCLUDED.obj_class_desc,
|
||||||
|
app_value = EXCLUDED.app_value, rnv_value = EXCLUDED.rnv_value;
|
||||||
|
""")
|
||||||
|
print(f'launch_obj_class: {len(parts)}')
|
||||||
|
|
||||||
|
# monthly time series
|
||||||
|
d = load_json('launch', 'charts')
|
||||||
|
if d:
|
||||||
|
calc = d.get('calculationType', 'SQUARE')
|
||||||
|
parts = []
|
||||||
|
for y in d.get('repYears', []):
|
||||||
|
yr = y['repYear']
|
||||||
|
for m in y.get('repMonth', []):
|
||||||
|
parts.append(f"('{SNAP}', {esc(calc)}, {yr}, {integer(m['repMonth'])}, "
|
||||||
|
f"{num(m.get('appValue'))}, {num(m.get('rnvValue'))})")
|
||||||
|
if parts:
|
||||||
|
for i in range(0, len(parts), 500):
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_launch_monthly (snapshot_date, calc_type, rep_year, rep_month, app_value, rnv_value)
|
||||||
|
VALUES {','.join(parts[i:i+500])}
|
||||||
|
ON CONFLICT (snapshot_date, calc_type, rep_year, rep_month) DO UPDATE SET
|
||||||
|
app_value = EXCLUDED.app_value, rnv_value = EXCLUDED.rnv_value;
|
||||||
|
""")
|
||||||
|
print(f'launch_monthly: {len(parts)}')
|
||||||
|
|
||||||
|
# ── 2. SOLD READY ────────────────────────────────────────────────────────
|
||||||
|
rep_year, rep_month = 2026, 3
|
||||||
|
d = load_json('sold_ready', 'ready-construction_index__repMonth_3_repYear_2026')
|
||||||
|
if d:
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_sold_ready_index (snapshot_date, rep_year, rep_month, square_sum, sold_perc, sold_sum, sold_ready_perc, ready_perc)
|
||||||
|
VALUES ('{SNAP}', {rep_year}, {rep_month}, {num(d.get('squareSumIndex'))},
|
||||||
|
{num(d.get('soldPercIndex'))}, {num(d.get('soldSumIndex'))},
|
||||||
|
{num(d.get('soldReadyPercIndex'))}, {num(d.get('readyPercIndex'))})
|
||||||
|
ON CONFLICT (snapshot_date, rep_year, rep_month) DO UPDATE SET
|
||||||
|
square_sum = EXCLUDED.square_sum, sold_perc = EXCLUDED.sold_perc,
|
||||||
|
sold_sum = EXCLUDED.sold_sum, sold_ready_perc = EXCLUDED.sold_ready_perc,
|
||||||
|
ready_perc = EXCLUDED.ready_perc;
|
||||||
|
""")
|
||||||
|
print('sold_ready_index: 1')
|
||||||
|
|
||||||
|
d = load_json('sold_ready', 'ready-construction_charts__repMonth_3_repYear_2026')
|
||||||
|
if d:
|
||||||
|
parts = []
|
||||||
|
seen = set()
|
||||||
|
for chart in d.get('charts', []):
|
||||||
|
ct = chart.get('chartType')
|
||||||
|
for row in chart.get('data', []):
|
||||||
|
key = row.get('key', '')
|
||||||
|
if not key: continue
|
||||||
|
pk = (ct, key)
|
||||||
|
if pk in seen: continue
|
||||||
|
seen.add(pk)
|
||||||
|
extra = {k: v for k, v in row.items() if k not in
|
||||||
|
{'key', 'squareSum', 'soldPerc', 'readyPerc', 'soldReadyPerc', 'foCd'}}
|
||||||
|
extra_json = json.dumps(extra, ensure_ascii=False) if extra else None
|
||||||
|
parts.append(f"('{SNAP}', {rep_year}, {rep_month}, {esc(ct)}, {esc(key)}, "
|
||||||
|
f"{num(row.get('squareSum'))}, {num(row.get('soldPerc'))}, "
|
||||||
|
f"{num(row.get('readyPerc'))}, {num(row.get('soldReadyPerc'))}, "
|
||||||
|
f"{integer(row.get('foCd'))}, "
|
||||||
|
f"{('NULL' if extra_json is None else esc(extra_json) + '::jsonb')})")
|
||||||
|
if parts:
|
||||||
|
for i in range(0, len(parts), 500):
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_sold_ready_breakdown (snapshot_date, rep_year, rep_month, chart_type, entity_key, square_sum, sold_perc, ready_perc, sold_ready_perc, fo_cd, extra)
|
||||||
|
VALUES {','.join(parts[i:i+500])}
|
||||||
|
ON CONFLICT (snapshot_date, rep_year, rep_month, chart_type, entity_key) DO UPDATE SET
|
||||||
|
square_sum = EXCLUDED.square_sum, sold_perc = EXCLUDED.sold_perc,
|
||||||
|
ready_perc = EXCLUDED.ready_perc, sold_ready_perc = EXCLUDED.sold_ready_perc,
|
||||||
|
fo_cd = EXCLUDED.fo_cd, extra = EXCLUDED.extra;
|
||||||
|
""")
|
||||||
|
print(f'sold_ready_breakdown: {len(parts)}')
|
||||||
|
|
||||||
|
d = load_json('sold_ready', 'ready-construction_dynamics__dynamicChartType_readyPercChart_2CsoldPercChart_2CsoldReadyPercChart_2CsquareSum')
|
||||||
|
if d:
|
||||||
|
parts = []
|
||||||
|
for chart in d.get('dynamicCharts', []):
|
||||||
|
ct = chart.get('dynamicChartType')
|
||||||
|
for y in chart.get('repYears', []):
|
||||||
|
yr = y.get('repYear')
|
||||||
|
for m in y.get('repMonths', []):
|
||||||
|
parts.append(f"('{SNAP}', {esc(ct)}, {integer(yr)}, {integer(m.get('repMonth'))}, {num(m.get('value'))})")
|
||||||
|
if parts:
|
||||||
|
for i in range(0, len(parts), 500):
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_sold_ready_dynamics (snapshot_date, dynamic_chart_type, rep_year, rep_month, value)
|
||||||
|
VALUES {','.join(parts[i:i+500])}
|
||||||
|
ON CONFLICT (snapshot_date, dynamic_chart_type, rep_year, rep_month) DO UPDATE SET
|
||||||
|
value = EXCLUDED.value;
|
||||||
|
""")
|
||||||
|
print(f'sold_ready_dynamics: {len(parts)}')
|
||||||
|
|
||||||
|
# ── 3. PROJECT FINANCE (share_construction) ─────────────────────────────
|
||||||
|
d = load_json('share_construction', 'dashboard') or load_json('share_construction', 'project_finance_dashboard__dateDay_26-04-2026')
|
||||||
|
if d:
|
||||||
|
rows = d.get('data', {}).get('data', []) if isinstance(d.get('data'), dict) else d.get('data', [])
|
||||||
|
parts = []
|
||||||
|
for row in rows:
|
||||||
|
extra = {k: v for k, v in row.items() if k not in {
|
||||||
|
'id','subjectType','subject','regionCd','foCd','foDesc',
|
||||||
|
'rnsCnt','devCnt','livSqAmt',
|
||||||
|
'guarantyEscrowRnsCnt','guarantyEscrowDevCnt','guarantyEscrowLivSqAmt',
|
||||||
|
'guarantyZosgRnsCnt','guarantyZosgDevCnt','guarantyZosgLivSqAmt',
|
||||||
|
'guarantyRnsCnt','guarantyDevCnt','guarantyLivSqAmt',
|
||||||
|
'nonguarantyRnsCnt','nonguarantyDevCnt','nonguarantyLivSqAmt'}}
|
||||||
|
parts.append((
|
||||||
|
row.get('subjectType'), row.get('subject'),
|
||||||
|
row.get('regionCd'), row.get('foCd'), row.get('foDesc'),
|
||||||
|
row.get('rnsCnt'), row.get('devCnt'), row.get('livSqAmt'),
|
||||||
|
row.get('guarantyEscrowRnsCnt'), row.get('guarantyEscrowDevCnt'), row.get('guarantyEscrowLivSqAmt'),
|
||||||
|
row.get('guarantyZosgRnsCnt'), row.get('guarantyZosgDevCnt'), row.get('guarantyZosgLivSqAmt'),
|
||||||
|
row.get('guarantyRnsCnt'), row.get('guarantyDevCnt'), row.get('guarantyLivSqAmt'),
|
||||||
|
row.get('nonguarantyRnsCnt'), row.get('nonguarantyDevCnt'), row.get('nonguarantyLivSqAmt'),
|
||||||
|
json.dumps(extra, ensure_ascii=False) if extra else None,
|
||||||
|
))
|
||||||
|
if parts:
|
||||||
|
# Clear today's data first (since PK is by subject_type+name, may have duplicates by region within fo)
|
||||||
|
psql(f"DELETE FROM domrf_project_finance WHERE snapshot_date = '{SNAP}';")
|
||||||
|
sql_parts = []
|
||||||
|
for p in parts:
|
||||||
|
(st, sn, rc, fc, fd, rn, dn, ls, gers, gerd, gerls,
|
||||||
|
gzrs, gzrd, gzrls, grns, grdv, grls, nrns, nrdv, nrls, ej) = p
|
||||||
|
sql_parts.append(
|
||||||
|
f"('{SNAP}', NULL, {esc(st)}, {esc(sn)}, {integer(rc)}, {integer(fc)}, {esc(fd)}, "
|
||||||
|
f"{integer(rn)}, {integer(dn)}, {num(ls)}, "
|
||||||
|
f"{integer(gers)}, {integer(gerd)}, {num(gerls)}, "
|
||||||
|
f"{integer(gzrs)}, {integer(gzrd)}, {num(gzrls)}, "
|
||||||
|
f"{integer(grns)}, {integer(grdv)}, {num(grls)}, "
|
||||||
|
f"{integer(nrns)}, {integer(nrdv)}, {num(nrls)}, "
|
||||||
|
f"{('NULL' if ej is None else esc(ej) + '::jsonb')})"
|
||||||
|
)
|
||||||
|
for i in range(0, len(sql_parts), 200):
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_project_finance (snapshot_date, report_date, subject_type, subject_name,
|
||||||
|
region_cd, fo_cd, fo_desc, rns_cnt, dev_cnt, liv_sq_amt,
|
||||||
|
guaranty_escrow_rns_cnt, guaranty_escrow_dev_cnt, guaranty_escrow_liv_sq_amt,
|
||||||
|
guaranty_zosg_rns_cnt, guaranty_zosg_dev_cnt, guaranty_zosg_liv_sq_amt,
|
||||||
|
guaranty_rns_cnt, guaranty_dev_cnt, guaranty_liv_sq_amt,
|
||||||
|
nonguaranty_rns_cnt, nonguaranty_dev_cnt, nonguaranty_liv_sq_amt, extra)
|
||||||
|
VALUES {','.join(sql_parts[i:i+200])}
|
||||||
|
ON CONFLICT DO NOTHING;
|
||||||
|
""")
|
||||||
|
print(f'project_finance: {len(parts)}')
|
||||||
|
|
||||||
|
# ── 4. COMMISSIONING ───────────────────────────────────────────────────
|
||||||
|
d = load_json('commissioning', 'building_summary_data')
|
||||||
|
if d:
|
||||||
|
rows = d.get('data', [])
|
||||||
|
parts = []
|
||||||
|
seen = set()
|
||||||
|
for r in rows:
|
||||||
|
pk = (r.get('regionId'), r.get('reportingPeriod'))
|
||||||
|
if pk in seen: continue
|
||||||
|
seen.add(pk)
|
||||||
|
extra = {k: v for k, v in r.items() if k not in {
|
||||||
|
'regionId','regionName','regionType','reportingPeriod','year',
|
||||||
|
'accumulatedFactArea','accumulatedFactAreaMultifamily','accumulatedFactAreaPrivate',
|
||||||
|
'accumulatedFactAreaChange','accumulatedFactAreaChangeShare'}}
|
||||||
|
parts.append(f"('{SNAP}', {integer(r.get('regionId'))}, {esc(r.get('regionName'))}, "
|
||||||
|
f"{esc(r.get('regionType'))}, {esc(r.get('reportingPeriod'))}::date, "
|
||||||
|
f"{integer(r.get('year'))}, "
|
||||||
|
f"{num(r.get('accumulatedFactArea'))}, "
|
||||||
|
f"{num(r.get('accumulatedFactAreaMultifamily'))}, "
|
||||||
|
f"{num(r.get('accumulatedFactAreaPrivate'))}, "
|
||||||
|
f"{num(r.get('accumulatedFactAreaChange'))}, "
|
||||||
|
f"{num(r.get('accumulatedFactAreaChangeShare'))}, "
|
||||||
|
f"{esc(json.dumps(extra, ensure_ascii=False)) + '::jsonb' if extra else 'NULL'})")
|
||||||
|
if parts:
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_commissioning (snapshot_date, region_id, region_name, region_type,
|
||||||
|
reporting_period, rep_year, accumulated_fact_area, accumulated_fact_area_multifamily,
|
||||||
|
accumulated_fact_area_private, accumulated_fact_area_change, accumulated_fact_area_change_share, extra)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, region_id, reporting_period) DO UPDATE SET
|
||||||
|
region_name = EXCLUDED.region_name,
|
||||||
|
accumulated_fact_area = EXCLUDED.accumulated_fact_area,
|
||||||
|
accumulated_fact_area_multifamily = EXCLUDED.accumulated_fact_area_multifamily,
|
||||||
|
accumulated_fact_area_private = EXCLUDED.accumulated_fact_area_private,
|
||||||
|
accumulated_fact_area_change = EXCLUDED.accumulated_fact_area_change,
|
||||||
|
accumulated_fact_area_change_share = EXCLUDED.accumulated_fact_area_change_share;
|
||||||
|
""")
|
||||||
|
print(f'commissioning: {len(parts)}')
|
||||||
|
|
||||||
|
# ── 5. MORTGAGE STATS ──────────────────────────────────────────────────
|
||||||
|
d = load_json('mortgage_stats', 'mortgage_dashboard_general')
|
||||||
|
if d and d.get('data'):
|
||||||
|
m = d['data']
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_mortgage_dashboard (snapshot_date,
|
||||||
|
total_credit_count, total_credit_count_delta_pct,
|
||||||
|
primary_credit_count, primary_credit_count_delta_pct,
|
||||||
|
secondary_credit_count, secondary_credit_count_delta_pct,
|
||||||
|
total_credit_amount, total_credit_amount_delta_pct,
|
||||||
|
primary_credit_amount, primary_credit_amount_delta_pct,
|
||||||
|
secondary_credit_amount, secondary_credit_amount_delta_pct,
|
||||||
|
total_credit_avg_rate, total_credit_avg_rate_delta,
|
||||||
|
primary_credit_avg_rate, primary_credit_avg_rate_delta,
|
||||||
|
secondary_credit_avg_rate, secondary_credit_avg_rate_delta)
|
||||||
|
VALUES ('{SNAP}',
|
||||||
|
{integer(m.get('totalCreditCount'))}, {num(m.get('totalCreditCountDeltaPct'))},
|
||||||
|
{integer(m.get('primaryCreditCount'))}, {num(m.get('primaryCreditCountDeltaPct'))},
|
||||||
|
{integer(m.get('secondaryCreditCount'))}, {num(m.get('secondaryCreditCountDeltaPct'))},
|
||||||
|
{num(m.get('totalCreditAmount'))}, {num(m.get('totalCreditAmountDeltaPct'))},
|
||||||
|
{num(m.get('primaryCreditAmount'))}, {num(m.get('primaryCreditAmountDeltaPct'))},
|
||||||
|
{num(m.get('secondaryCreditAmount'))}, {num(m.get('secondaryCreditAmountDeltaPct'))},
|
||||||
|
{num(m.get('totalCreditAvgRate'))}, {num(m.get('totalCreditAvgRateDelta'))},
|
||||||
|
{num(m.get('primaryCreditAvgRate'))}, {num(m.get('primaryCreditAvgRateDelta'))},
|
||||||
|
{num(m.get('secondaryCreditAvgRate'))}, {num(m.get('secondaryCreditAvgRateDelta'))})
|
||||||
|
ON CONFLICT (snapshot_date) DO UPDATE SET
|
||||||
|
total_credit_count = EXCLUDED.total_credit_count,
|
||||||
|
total_credit_amount = EXCLUDED.total_credit_amount,
|
||||||
|
total_credit_avg_rate = EXCLUDED.total_credit_avg_rate;
|
||||||
|
""")
|
||||||
|
print('mortgage_dashboard: 1')
|
||||||
|
|
||||||
|
d = load_json('mortgage_stats', 'mortgage_dashboard_details')
|
||||||
|
if d and d.get('data'):
|
||||||
|
rows = d['data']
|
||||||
|
parts = [f"('{SNAP}', {esc(r.get('currency'))}, {num(r.get('creditAmountAvg'))}, "
|
||||||
|
f"{num(r.get('creditAmountAvgDeltaPct'))}, {num(r.get('creditAvgPeriod'))}, "
|
||||||
|
f"{num(r.get('creditAvgPeriodDeltaPct'))}, {num(r.get('creditDebtsAmount'))}, "
|
||||||
|
f"{num(r.get('creditDebtsAmountDeltaPct'))}, {num(r.get('creditDebtsOverduePercent'))}, "
|
||||||
|
f"{num(r.get('creditDebtsOverduePercentDelta'))})"
|
||||||
|
for r in rows]
|
||||||
|
if parts:
|
||||||
|
psql(f"""
|
||||||
|
INSERT INTO domrf_mortgage_details (snapshot_date, currency, credit_amount_avg,
|
||||||
|
credit_amount_avg_delta_pct, credit_avg_period, credit_avg_period_delta_pct,
|
||||||
|
credit_debts_amount, credit_debts_amount_delta_pct,
|
||||||
|
credit_debts_overdue_percent, credit_debts_overdue_percent_delta)
|
||||||
|
VALUES {','.join(parts)}
|
||||||
|
ON CONFLICT (snapshot_date, currency) DO UPDATE SET
|
||||||
|
credit_amount_avg = EXCLUDED.credit_amount_avg,
|
||||||
|
credit_avg_period = EXCLUDED.credit_avg_period,
|
||||||
|
credit_debts_amount = EXCLUDED.credit_debts_amount;
|
||||||
|
""")
|
||||||
|
print(f'mortgage_details: {len(parts)}')
|
||||||
|
|
||||||
|
print('\n--- summary ---')
|
||||||
|
out = psql(f"""
|
||||||
|
SELECT 'launch_top' tab, COUNT(*) FROM domrf_launch_top WHERE snapshot_date = '{SNAP}'
|
||||||
|
UNION ALL SELECT 'launch_obj_class', COUNT(*) FROM domrf_launch_obj_class WHERE snapshot_date = '{SNAP}'
|
||||||
|
UNION ALL SELECT 'launch_monthly', COUNT(*) FROM domrf_launch_monthly WHERE snapshot_date = '{SNAP}'
|
||||||
|
UNION ALL SELECT 'sold_ready_index', COUNT(*) FROM domrf_sold_ready_index WHERE snapshot_date = '{SNAP}'
|
||||||
|
UNION ALL SELECT 'sold_ready_breakdown', COUNT(*) FROM domrf_sold_ready_breakdown WHERE snapshot_date = '{SNAP}'
|
||||||
|
UNION ALL SELECT 'sold_ready_dynamics', COUNT(*) FROM domrf_sold_ready_dynamics WHERE snapshot_date = '{SNAP}'
|
||||||
|
UNION ALL SELECT 'project_finance', COUNT(*) FROM domrf_project_finance WHERE snapshot_date = '{SNAP}'
|
||||||
|
UNION ALL SELECT 'commissioning', COUNT(*) FROM domrf_commissioning WHERE snapshot_date = '{SNAP}'
|
||||||
|
UNION ALL SELECT 'mortgage_dashboard', COUNT(*) FROM domrf_mortgage_dashboard WHERE snapshot_date = '{SNAP}'
|
||||||
|
UNION ALL SELECT 'mortgage_details', COUNT(*) FROM domrf_mortgage_details WHERE snapshot_date = '{SNAP}'
|
||||||
|
ORDER BY 1;
|
||||||
|
""")
|
||||||
|
print(out)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
99
data/sql/32_load_domrf_raw.py
Normal file
99
data/sql/32_load_domrf_raw.py
Normal file
|
|
@ -0,0 +1,99 @@
|
||||||
|
"""Universal loader: dumps every scraped JSON from data/raw/domrf_full/ into Postgres
|
||||||
|
as JSONB rows in domrf_raw_endpoints. Idempotent (UPSERT by snapshot+section+endpoint).
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... python data/sql/32_load_domrf_raw.py
|
||||||
|
"""
|
||||||
|
import os, json, subprocess, sys, datetime
|
||||||
|
|
||||||
|
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||||
|
ROOT = os.path.join(HERE, '..', 'raw', 'domrf_full')
|
||||||
|
PG_PORT = os.environ.get('PG_PORT', '5432')
|
||||||
|
PG_USER = os.environ.get('PG_USER', 'gendesign')
|
||||||
|
PG_DB = os.environ.get('PG_DB', 'gendesign')
|
||||||
|
PG_PASS = os.environ.get('PGPASSWORD', '')
|
||||||
|
SNAP = os.environ.get('SNAPSHOT_DATE', datetime.date.today().isoformat())
|
||||||
|
|
||||||
|
|
||||||
|
def psql(sql):
|
||||||
|
cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
|
||||||
|
'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT,
|
||||||
|
'-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet']
|
||||||
|
res = subprocess.run(cmd, input=sql, capture_output=True, text=True, encoding='utf-8')
|
||||||
|
if res.returncode != 0:
|
||||||
|
print('psql FAILED:', res.stderr[-1500:], file=sys.stderr)
|
||||||
|
raise SystemExit(res.returncode)
|
||||||
|
return res.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def psql_stdin(sql, stdin_data):
|
||||||
|
"""Use psql with COPY via stdin to load big JSONs (avoids cmdline length limits)."""
|
||||||
|
cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
|
||||||
|
'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT,
|
||||||
|
'-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet']
|
||||||
|
res = subprocess.run(cmd, input=stdin_data, capture_output=True, text=True, encoding='utf-8')
|
||||||
|
if res.returncode != 0:
|
||||||
|
print('psql FAILED:', res.stderr[-1500:], file=sys.stderr)
|
||||||
|
raise SystemExit(res.returncode)
|
||||||
|
return res.stdout
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
schema = open(os.path.join(HERE, '31_schema_domrf_raw.sql'), encoding='utf-8').read()
|
||||||
|
psql(schema)
|
||||||
|
print('schema applied')
|
||||||
|
|
||||||
|
loaded = 0
|
||||||
|
skipped = 0
|
||||||
|
by_section = {}
|
||||||
|
|
||||||
|
for section in sorted(os.listdir(ROOT)):
|
||||||
|
sec_dir = os.path.join(ROOT, section)
|
||||||
|
if not os.path.isdir(sec_dir):
|
||||||
|
continue
|
||||||
|
files = [f for f in os.listdir(sec_dir) if f.endswith('.json')]
|
||||||
|
for fname in sorted(files):
|
||||||
|
fpath = os.path.join(sec_dir, fname)
|
||||||
|
try:
|
||||||
|
text = open(fpath, encoding='utf-8').read()
|
||||||
|
json.loads(text) # validate
|
||||||
|
except Exception as e:
|
||||||
|
print(f' SKIP invalid JSON: {section}/{fname}: {e}')
|
||||||
|
skipped += 1
|
||||||
|
continue
|
||||||
|
endpoint = fname[:-5] # strip .json
|
||||||
|
# Use COPY ... FROM STDIN for safe binary insertion
|
||||||
|
# Use psycopg-style escape: \\ → \\\\, replace tabs/newlines in JSON keys
|
||||||
|
# Easier: pass JSON via stdin to a Python heredoc-style INSERT
|
||||||
|
sql = f"""
|
||||||
|
INSERT INTO domrf_raw_endpoints (snapshot_date, section, endpoint, source_url, payload, payload_size, fetched_at)
|
||||||
|
VALUES ('{SNAP}', $sec${section}$sec$, $end${endpoint}$end$, NULL, $j${text}$j$::jsonb, {len(text)}, NOW())
|
||||||
|
ON CONFLICT (snapshot_date, section, endpoint) DO UPDATE SET
|
||||||
|
payload = EXCLUDED.payload,
|
||||||
|
payload_size = EXCLUDED.payload_size,
|
||||||
|
fetched_at = NOW();
|
||||||
|
"""
|
||||||
|
psql_stdin('', sql)
|
||||||
|
loaded += 1
|
||||||
|
by_section.setdefault(section, 0)
|
||||||
|
by_section[section] += 1
|
||||||
|
|
||||||
|
print(f'\nLOADED {loaded} files, SKIPPED {skipped}')
|
||||||
|
print('--- by section ---')
|
||||||
|
for s, n in sorted(by_section.items()):
|
||||||
|
print(f' {s:22s} {n}')
|
||||||
|
|
||||||
|
# Final summary from DB
|
||||||
|
print('\n--- DB summary ---')
|
||||||
|
out = psql(f"""
|
||||||
|
SELECT section, COUNT(*) AS endpoints, pg_size_pretty(SUM(payload_size)::bigint) AS total
|
||||||
|
FROM domrf_raw_endpoints
|
||||||
|
WHERE snapshot_date = '{SNAP}'
|
||||||
|
GROUP BY section
|
||||||
|
ORDER BY section;
|
||||||
|
""")
|
||||||
|
print(out)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
main()
|
||||||
287
data/sql/40_relations.sql
Normal file
287
data/sql/40_relations.sql
Normal file
|
|
@ -0,0 +1,287 @@
|
||||||
|
-- Foreign-key constraints + helper VIEWs across DOM.RF + Rosreestr tables.
|
||||||
|
-- Idempotent: bootstraps missing dimension rows first, then ALTER TABLE ... ADD CONSTRAINT IF NOT EXISTS.
|
||||||
|
|
||||||
|
-- ─────────────────────────────────────────────────────────────────────────
|
||||||
|
-- 1. BOOTSTRAP DIMENSION TABLES
|
||||||
|
-- ─────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
-- Add row 0 = РФ to domrf_regions (used as "all-Russia" placeholder by some tables)
|
||||||
|
INSERT INTO domrf_regions (region_id, region_name, federal_district)
|
||||||
|
VALUES (0, 'Российская Федерация', '_RF')
|
||||||
|
ON CONFLICT (region_id) DO NOTHING;
|
||||||
|
|
||||||
|
-- Backfill domrf_snapshots from every snapshot_date currently used in children
|
||||||
|
INSERT INTO domrf_snapshots (snapshot_date, notes)
|
||||||
|
SELECT DISTINCT s, 'auto-backfilled by 40_relations.sql' FROM (
|
||||||
|
SELECT snapshot_date AS s FROM domrf_developers_full
|
||||||
|
UNION SELECT snapshot_date FROM domrf_escrow_banks
|
||||||
|
UNION SELECT snapshot_date FROM domrf_guaranty_regions
|
||||||
|
UNION SELECT snapshot_date FROM domrf_housing_summary
|
||||||
|
UNION SELECT snapshot_date FROM domrf_planned_commissioning
|
||||||
|
UNION SELECT snapshot_date FROM domrf_sold_out
|
||||||
|
UNION SELECT snapshot_date FROM domrf_sold_out_by_year
|
||||||
|
UNION SELECT snapshot_date FROM domrf_sold_out_by_progress
|
||||||
|
UNION SELECT snapshot_date FROM domrf_mortgage_rates
|
||||||
|
UNION SELECT snapshot_date FROM domrf_region_aggregates
|
||||||
|
UNION SELECT snapshot_date FROM domrf_flat_area_distribution
|
||||||
|
UNION SELECT snapshot_date FROM domrf_developer_aggregates
|
||||||
|
UNION SELECT snapshot_date FROM domrf_launch_top
|
||||||
|
UNION SELECT snapshot_date FROM domrf_launch_obj_class
|
||||||
|
UNION SELECT snapshot_date FROM domrf_launch_monthly
|
||||||
|
UNION SELECT snapshot_date FROM domrf_sold_ready_index
|
||||||
|
UNION SELECT snapshot_date FROM domrf_sold_ready_breakdown
|
||||||
|
UNION SELECT snapshot_date FROM domrf_sold_ready_dynamics
|
||||||
|
UNION SELECT snapshot_date FROM domrf_project_finance
|
||||||
|
UNION SELECT snapshot_date FROM domrf_commissioning
|
||||||
|
UNION SELECT snapshot_date FROM domrf_mortgage_dashboard
|
||||||
|
UNION SELECT snapshot_date FROM domrf_mortgage_details
|
||||||
|
) AS dates
|
||||||
|
ON CONFLICT (snapshot_date) DO NOTHING;
|
||||||
|
|
||||||
|
-- Helper to add a constraint only if it does not already exist
|
||||||
|
-- (PostgreSQL ≥9.6: do via DO block; idempotent)
|
||||||
|
DO $$
|
||||||
|
DECLARE
|
||||||
|
rec RECORD;
|
||||||
|
BEGIN
|
||||||
|
-- ──────────────────────────────────────────────────────────────────────
|
||||||
|
-- 2. SNAPSHOT FKs (every domrf_* with snapshot_date column)
|
||||||
|
-- ──────────────────────────────────────────────────────────────────────
|
||||||
|
FOR rec IN SELECT * FROM (VALUES
|
||||||
|
('domrf_developers_full', 'fk_snap_developers_full'),
|
||||||
|
('domrf_escrow_banks', 'fk_snap_escrow_banks'),
|
||||||
|
('domrf_guaranty_regions', 'fk_snap_guaranty_regions'),
|
||||||
|
('domrf_housing_summary', 'fk_snap_housing_summary'),
|
||||||
|
('domrf_planned_commissioning', 'fk_snap_planned_commissioning'),
|
||||||
|
('domrf_sold_out', 'fk_snap_sold_out'),
|
||||||
|
('domrf_sold_out_by_year', 'fk_snap_sold_out_by_year'),
|
||||||
|
('domrf_sold_out_by_progress', 'fk_snap_sold_out_by_progress'),
|
||||||
|
('domrf_mortgage_rates', 'fk_snap_mortgage_rates'),
|
||||||
|
('domrf_region_aggregates', 'fk_snap_region_aggregates'),
|
||||||
|
('domrf_flat_area_distribution', 'fk_snap_flat_area_distribution'),
|
||||||
|
('domrf_developer_aggregates', 'fk_snap_developer_aggregates'),
|
||||||
|
('domrf_launch_top', 'fk_snap_launch_top'),
|
||||||
|
('domrf_launch_obj_class', 'fk_snap_launch_obj_class'),
|
||||||
|
('domrf_launch_monthly', 'fk_snap_launch_monthly'),
|
||||||
|
('domrf_sold_ready_index', 'fk_snap_sold_ready_index'),
|
||||||
|
('domrf_sold_ready_breakdown', 'fk_snap_sold_ready_breakdown'),
|
||||||
|
('domrf_sold_ready_dynamics', 'fk_snap_sold_ready_dynamics'),
|
||||||
|
('domrf_project_finance', 'fk_snap_project_finance'),
|
||||||
|
('domrf_commissioning', 'fk_snap_commissioning'),
|
||||||
|
('domrf_mortgage_dashboard', 'fk_snap_mortgage_dashboard'),
|
||||||
|
('domrf_mortgage_details', 'fk_snap_mortgage_details'),
|
||||||
|
('domrf_raw_endpoints', 'fk_snap_raw_endpoints')
|
||||||
|
) AS t(tbl, cn)
|
||||||
|
LOOP
|
||||||
|
IF NOT EXISTS (
|
||||||
|
SELECT 1 FROM pg_constraint WHERE conname = rec.cn
|
||||||
|
) THEN
|
||||||
|
EXECUTE format(
|
||||||
|
'ALTER TABLE %I ADD CONSTRAINT %I FOREIGN KEY (snapshot_date) REFERENCES domrf_snapshots(snapshot_date) ON UPDATE CASCADE',
|
||||||
|
rec.tbl, rec.cn
|
||||||
|
);
|
||||||
|
END IF;
|
||||||
|
END LOOP;
|
||||||
|
|
||||||
|
-- ──────────────────────────────────────────────────────────────────────
|
||||||
|
-- 3. REGION FKs (domrf_region_aggregates / flat_area_distribution / developer_aggregates → domrf_regions)
|
||||||
|
-- ──────────────────────────────────────────────────────────────────────
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_region_aggregates_region') THEN
|
||||||
|
ALTER TABLE domrf_region_aggregates
|
||||||
|
ADD CONSTRAINT fk_region_aggregates_region
|
||||||
|
FOREIGN KEY (region_id) REFERENCES domrf_regions(region_id) ON UPDATE CASCADE;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_flat_area_region') THEN
|
||||||
|
ALTER TABLE domrf_flat_area_distribution
|
||||||
|
ADD CONSTRAINT fk_flat_area_region
|
||||||
|
FOREIGN KEY (region_id) REFERENCES domrf_regions(region_id) ON UPDATE CASCADE;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_developer_agg_region') THEN
|
||||||
|
ALTER TABLE domrf_developer_aggregates
|
||||||
|
ADD CONSTRAINT fk_developer_agg_region
|
||||||
|
FOREIGN KEY (region_id) REFERENCES domrf_regions(region_id) ON UPDATE CASCADE;
|
||||||
|
END IF;
|
||||||
|
|
||||||
|
-- ──────────────────────────────────────────────────────────────────────
|
||||||
|
-- 4. DEVELOPER FK (domrf_developer_aggregates → domrf_developers)
|
||||||
|
-- ──────────────────────────────────────────────────────────────────────
|
||||||
|
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_developer_agg_developer') THEN
|
||||||
|
ALTER TABLE domrf_developer_aggregates
|
||||||
|
ADD CONSTRAINT fk_developer_agg_developer
|
||||||
|
FOREIGN KEY (developer_id) REFERENCES domrf_developers(developer_id) ON UPDATE CASCADE;
|
||||||
|
END IF;
|
||||||
|
END $$;
|
||||||
|
|
||||||
|
-- ─────────────────────────────────────────────────────────────────────────
|
||||||
|
-- 5. HELPER VIEWS
|
||||||
|
-- ─────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
-- v_region_master: одна строка на регион — все ключевые метрики на последний снапшот
|
||||||
|
DROP VIEW IF EXISTS v_region_master CASCADE;
|
||||||
|
CREATE VIEW v_region_master AS
|
||||||
|
WITH latest AS (SELECT MAX(snapshot_date) AS d FROM domrf_snapshots)
|
||||||
|
SELECT
|
||||||
|
r.region_id,
|
||||||
|
r.region_name,
|
||||||
|
r.federal_district,
|
||||||
|
-- Quartirografia totals (TOTAL row)
|
||||||
|
rta.flat_count AS total_flats,
|
||||||
|
rta.area_sqm AS total_area_sqm,
|
||||||
|
-- Sold-out (DOM.RF PDF data)
|
||||||
|
so.total_area_th_sqm AS sold_out_total_th_sqm,
|
||||||
|
so.sold_pct AS sold_out_sold_pct,
|
||||||
|
so.unsold_pct AS sold_out_unsold_pct,
|
||||||
|
-- Sold-ready breakdown (Playwright API data)
|
||||||
|
srb.square_sum AS sold_ready_square,
|
||||||
|
srb.sold_perc AS sold_ready_sold_pct,
|
||||||
|
srb.ready_perc AS sold_ready_ready_pct,
|
||||||
|
srb.sold_ready_perc AS sold_ready_balance_pct,
|
||||||
|
-- Project finance
|
||||||
|
pf.rns_cnt AS project_finance_permits,
|
||||||
|
pf.dev_cnt AS project_finance_developers,
|
||||||
|
pf.guaranty_escrow_dev_cnt AS project_finance_escrow_devs,
|
||||||
|
-- Guaranty (XLSX)
|
||||||
|
g.fz214_developers AS guaranty_fz214_devs,
|
||||||
|
g.escrow_developers AS guaranty_escrow_devs,
|
||||||
|
-- Launch (запуски новых)
|
||||||
|
(SELECT value FROM domrf_launch_top
|
||||||
|
WHERE dim_type = 'region' AND metric_type = 'app'
|
||||||
|
AND entity_id = r.region_id::text AND snapshot_date = (SELECT d FROM latest)
|
||||||
|
LIMIT 1) AS launch_app_2026_sqm,
|
||||||
|
(SELECT value FROM domrf_launch_top
|
||||||
|
WHERE dim_type = 'region' AND metric_type = 'rnv'
|
||||||
|
AND entity_id = r.region_id::text AND snapshot_date = (SELECT d FROM latest)
|
||||||
|
LIMIT 1) AS launch_rnv_2026_sqm
|
||||||
|
FROM domrf_regions r
|
||||||
|
LEFT JOIN domrf_region_aggregates rta
|
||||||
|
ON rta.region_id = r.region_id
|
||||||
|
AND rta.room_count_type = 'TOTAL'
|
||||||
|
AND rta.snapshot_date = (SELECT d FROM latest)
|
||||||
|
LEFT JOIN domrf_sold_out so
|
||||||
|
ON so.territory_name = r.region_name
|
||||||
|
AND so.snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_sold_out WHERE territory_name = r.region_name)
|
||||||
|
LEFT JOIN domrf_sold_ready_breakdown srb
|
||||||
|
ON srb.entity_key = r.region_name
|
||||||
|
AND srb.chart_type = 'regionChart'
|
||||||
|
AND srb.snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_sold_ready_breakdown WHERE chart_type='regionChart')
|
||||||
|
LEFT JOIN domrf_project_finance pf
|
||||||
|
ON pf.subject_name = r.region_name
|
||||||
|
AND pf.snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_project_finance)
|
||||||
|
LEFT JOIN domrf_guaranty_regions g
|
||||||
|
ON g.territory_name = r.region_name
|
||||||
|
AND g.snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_guaranty_regions)
|
||||||
|
WHERE r.region_id > 0
|
||||||
|
ORDER BY total_area_sqm DESC NULLS LAST;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_region_master IS
|
||||||
|
'Один срез по региону: квартирография + распроданность (DOM.РФ + sold_out PDF) + проектное финансирование + запуски/ввод 2026';
|
||||||
|
|
||||||
|
-- v_developer_master: профиль застройщика (latest snapshot)
|
||||||
|
DROP VIEW IF EXISTS v_developer_master CASCADE;
|
||||||
|
CREATE VIEW v_developer_master AS
|
||||||
|
WITH latest AS (SELECT MAX(snapshot_date) AS d FROM domrf_snapshots)
|
||||||
|
SELECT
|
||||||
|
d.developer_id,
|
||||||
|
d.developer_name,
|
||||||
|
-- Group-of-companies aggregates (from JSON dashboard)
|
||||||
|
dat.flat_count AS group_flat_count,
|
||||||
|
dat.area_sqm AS group_area_sqm,
|
||||||
|
-- Per-юрлицо реестр (XLSX)
|
||||||
|
df.area_thousand_sqm AS legal_area_th_sqm,
|
||||||
|
df.permits_count AS legal_permits,
|
||||||
|
df.houses_count AS legal_houses,
|
||||||
|
df.flats_count AS legal_flats,
|
||||||
|
df.market_share_pct AS market_share_pct,
|
||||||
|
-- Запуски / ввод 2026 (из launch_top, dim=developer)
|
||||||
|
(SELECT value FROM domrf_launch_top
|
||||||
|
WHERE dim_type='developer' AND metric_type='app'
|
||||||
|
AND entity_id = d.developer_id AND snapshot_date = (SELECT d FROM latest)
|
||||||
|
LIMIT 1) AS launch_app_2026_sqm,
|
||||||
|
(SELECT value FROM domrf_launch_top
|
||||||
|
WHERE dim_type='developer' AND metric_type='rnv'
|
||||||
|
AND entity_id = d.developer_id AND snapshot_date = (SELECT d FROM latest)
|
||||||
|
LIMIT 1) AS launch_rnv_2026_sqm,
|
||||||
|
-- Распроданность (devChart)
|
||||||
|
(SELECT square_sum FROM domrf_sold_ready_breakdown
|
||||||
|
WHERE chart_type='devChart' AND entity_key = d.developer_name
|
||||||
|
AND snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_sold_ready_breakdown WHERE chart_type='devChart')
|
||||||
|
LIMIT 1) AS sold_ready_square,
|
||||||
|
(SELECT sold_perc FROM domrf_sold_ready_breakdown
|
||||||
|
WHERE chart_type='devChart' AND entity_key = d.developer_name
|
||||||
|
AND snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_sold_ready_breakdown WHERE chart_type='devChart')
|
||||||
|
LIMIT 1) AS sold_ready_sold_pct
|
||||||
|
FROM domrf_developers d
|
||||||
|
LEFT JOIN domrf_developer_aggregates dat
|
||||||
|
ON dat.developer_id = d.developer_id
|
||||||
|
AND dat.room_count_type = 'TOTAL'
|
||||||
|
AND dat.snapshot_date = (SELECT d FROM latest)
|
||||||
|
LEFT JOIN domrf_developers_full df
|
||||||
|
ON df.developer_name = d.developer_name
|
||||||
|
AND df.snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_developers_full WHERE developer_name = d.developer_name);
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_developer_master IS
|
||||||
|
'Профиль застройщика: реестр + квартирография по комнатностям + запуски/ввод + распроданность';
|
||||||
|
|
||||||
|
-- v_rosreestr_market_by_region — срез цен сделок и застройщиков по региону
|
||||||
|
DROP VIEW IF EXISTS v_rosreestr_market_by_region CASCADE;
|
||||||
|
CREATE VIEW v_rosreestr_market_by_region AS
|
||||||
|
SELECT
|
||||||
|
r.region_id,
|
||||||
|
r.region_name,
|
||||||
|
r.federal_district,
|
||||||
|
rar.period_start_date,
|
||||||
|
rar.doc_type,
|
||||||
|
rar.deals_count,
|
||||||
|
rar.median_price_sqm AS median_rub_sqm,
|
||||||
|
rar.p10_price_sqm AS p10_rub_sqm,
|
||||||
|
rar.p90_price_sqm AS p90_rub_sqm
|
||||||
|
FROM domrf_regions r
|
||||||
|
JOIN rr_agg_region rar
|
||||||
|
ON rar.region_code = r.region_id
|
||||||
|
WHERE rar.realestate_type_code = '002001003000' -- Помещения (квартиры)
|
||||||
|
ORDER BY r.region_id, rar.period_start_date, rar.doc_type;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_rosreestr_market_by_region IS
|
||||||
|
'Цены сделок Росреестра по регионам с DOM.РФ-именами регионов (квартиры)';
|
||||||
|
|
||||||
|
-- v_market_pulse_rf — РФ-уровень summary с разных источников за last snapshot
|
||||||
|
DROP VIEW IF EXISTS v_market_pulse_rf CASCADE;
|
||||||
|
CREATE VIEW v_market_pulse_rf AS
|
||||||
|
SELECT
|
||||||
|
-- Quartirografia
|
||||||
|
(SELECT SUM(area_sqm)/1e6 FROM domrf_region_aggregates
|
||||||
|
WHERE room_count_type='TOTAL'
|
||||||
|
AND snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_region_aggregates)) AS total_construction_mln_sqm,
|
||||||
|
-- Sold-ready
|
||||||
|
(SELECT square_sum/1e6 FROM domrf_sold_ready_index
|
||||||
|
ORDER BY snapshot_date DESC LIMIT 1) AS sold_ready_total_mln_sqm,
|
||||||
|
(SELECT sold_perc FROM domrf_sold_ready_index
|
||||||
|
ORDER BY snapshot_date DESC LIMIT 1) AS sold_pct,
|
||||||
|
(SELECT ready_perc FROM domrf_sold_ready_index
|
||||||
|
ORDER BY snapshot_date DESC LIMIT 1) AS ready_pct,
|
||||||
|
-- Прайс
|
||||||
|
(SELECT avg_price_per_sqm FROM domrf_sold_out
|
||||||
|
WHERE territory_name = 'Российская Федерация'
|
||||||
|
ORDER BY snapshot_date DESC LIMIT 1) AS rf_avg_price_per_sqm,
|
||||||
|
-- Ставки
|
||||||
|
(SELECT primary_rate FROM domrf_mortgage_rates
|
||||||
|
WHERE bank_name = 'Средневзвешенная'
|
||||||
|
ORDER BY snapshot_date DESC LIMIT 1) AS market_primary_rate,
|
||||||
|
(SELECT secondary_rate FROM domrf_mortgage_rates
|
||||||
|
WHERE bank_name = 'Средневзвешенная'
|
||||||
|
ORDER BY snapshot_date DESC LIMIT 1) AS market_secondary_rate,
|
||||||
|
-- Реальные ставки (по выданным кредитам)
|
||||||
|
(SELECT primary_credit_avg_rate FROM domrf_mortgage_dashboard
|
||||||
|
ORDER BY snapshot_date DESC LIMIT 1) AS actual_primary_rate,
|
||||||
|
(SELECT secondary_credit_avg_rate FROM domrf_mortgage_dashboard
|
||||||
|
ORDER BY snapshot_date DESC LIMIT 1) AS actual_secondary_rate,
|
||||||
|
-- Объём кредитования
|
||||||
|
(SELECT total_credit_count FROM domrf_mortgage_dashboard
|
||||||
|
ORDER BY snapshot_date DESC LIMIT 1) AS total_credit_count,
|
||||||
|
(SELECT total_credit_amount/1e9 FROM domrf_mortgage_dashboard
|
||||||
|
ORDER BY snapshot_date DESC LIMIT 1) AS total_credit_amount_bln_rub;
|
||||||
|
|
||||||
|
COMMENT ON VIEW v_market_pulse_rf IS
|
||||||
|
'РФ pulse: объём строительства, распроданность, средняя цена, ставки (рыночные vs реальные), объём ипотеки';
|
||||||
11
data/sql/decode_to_file.py
Normal file
11
data/sql/decode_to_file.py
Normal file
|
|
@ -0,0 +1,11 @@
|
||||||
|
"""Reads JSON-encoded string from stdin (the kind MCP returns when wrapping a string),
|
||||||
|
and writes the decoded value as UTF-8 to the given file."""
|
||||||
|
import json, sys, os
|
||||||
|
out_path = sys.argv[1]
|
||||||
|
raw = sys.stdin.read()
|
||||||
|
# raw is a JSON string like "[{\"name\":\"...\"}]" — already JSON-quoted
|
||||||
|
decoded = json.loads(raw)
|
||||||
|
os.makedirs(os.path.dirname(out_path) or '.', exist_ok=True)
|
||||||
|
with open(out_path, 'w', encoding='utf-8') as f:
|
||||||
|
f.write(decoded)
|
||||||
|
print(f'wrote {out_path} ({len(decoded)} chars)')
|
||||||
57
data/sql/upload_server.py
Normal file
57
data/sql/upload_server.py
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
"""Tiny HTTP receiver: POST <body> to /<path>, file gets written to data/raw/domrf/<path>.
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
python data/sql/upload_server.py # listens on 127.0.0.1:8765
|
||||||
|
|
||||||
|
In browser DevTools:
|
||||||
|
fetch('http://127.0.0.1:8765/regions.json', { method: 'POST', body: JSON.stringify(window.__domrf['regions']) });
|
||||||
|
"""
|
||||||
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
|
||||||
|
ROOT = os.path.join(os.path.dirname(__file__), '..', 'raw', 'domrf')
|
||||||
|
os.makedirs(ROOT, exist_ok=True)
|
||||||
|
|
||||||
|
|
||||||
|
class H(BaseHTTPRequestHandler):
|
||||||
|
def _cors(self):
|
||||||
|
self.send_header('Access-Control-Allow-Origin', '*')
|
||||||
|
self.send_header('Access-Control-Allow-Methods', 'POST, OPTIONS')
|
||||||
|
self.send_header('Access-Control-Allow-Headers', 'Content-Type')
|
||||||
|
|
||||||
|
def do_OPTIONS(self):
|
||||||
|
self.send_response(204)
|
||||||
|
self._cors()
|
||||||
|
self.end_headers()
|
||||||
|
|
||||||
|
def do_POST(self):
|
||||||
|
try:
|
||||||
|
length = int(self.headers.get('Content-Length', 0))
|
||||||
|
body = self.rfile.read(length)
|
||||||
|
name = self.path.lstrip('/').replace('..', '_').replace('\\', '_')
|
||||||
|
if not name:
|
||||||
|
name = 'upload.json'
|
||||||
|
path = os.path.join(ROOT, name)
|
||||||
|
with open(path, 'wb') as f:
|
||||||
|
f.write(body)
|
||||||
|
print(f'WROTE {path} ({len(body)} bytes)', flush=True)
|
||||||
|
self.send_response(200)
|
||||||
|
self._cors()
|
||||||
|
self.send_header('Content-Type', 'application/json')
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(b'{"ok":true}')
|
||||||
|
except Exception as e:
|
||||||
|
self.send_response(500)
|
||||||
|
self._cors()
|
||||||
|
self.end_headers()
|
||||||
|
self.wfile.write(str(e).encode())
|
||||||
|
|
||||||
|
def log_message(self, *args):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
addr = ('127.0.0.1', 8765)
|
||||||
|
print(f'listening on http://{addr[0]}:{addr[1]}, output dir: {os.path.abspath(ROOT)}', flush=True)
|
||||||
|
HTTPServer(addr, H).serve_forever()
|
||||||
File diff suppressed because one or more lines are too long
24
sshkey.txt
24
sshkey.txt
|
|
@ -56,3 +56,27 @@ Port: 15432
|
||||||
Database: gendesign
|
Database: gendesign
|
||||||
Username: gendesign
|
Username: gendesign
|
||||||
Password: 2J2SBPMKuS998fiwhtQqDhMI
|
Password: 2J2SBPMKuS998fiwhtQqDhMI
|
||||||
|
|
||||||
|
Локально установить pre-commit один раз:
|
||||||
|
make pre-commit-install
|
||||||
|
# или:
|
||||||
|
pip install pre-commit
|
||||||
|
pre-commit install
|
||||||
|
|
||||||
|
python -m pre_commit install
|
||||||
|
|
||||||
|
python -m pre_commit run --all-files
|
||||||
|
# или, если PATH добавлен:
|
||||||
|
pre-commit run --all-files
|
||||||
|
|
||||||
|
Только текущая сессия:
|
||||||
|
|
||||||
|
$env:PATH += ";$env:LOCALAPPDATA\Python\pythoncore-3.14-64\Scripts"
|
||||||
|
pre-commit install
|
||||||
|
Постоянно (нужно ОДИН раз):
|
||||||
|
|
||||||
|
[Environment]::SetEnvironmentVariable(
|
||||||
|
"PATH",
|
||||||
|
$env:PATH + ";$env:LOCALAPPDATA\Python\pythoncore-3.14-64\Scripts",
|
||||||
|
"User"
|
||||||
|
)
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue