add data
This commit is contained in:
lekss361
parent
a2dcd9844a
commit
c13cbaef2e
18 changed files with 2409 additions and 0 deletions
6
.gitignore
vendored
6
.gitignore
vendored
|
|
@ -21,3 +21,9 @@ node_modules/
|
|||
|
||||
# Claude Code local state
|
||||
.claude/
|
||||
|
||||
# Local data (raw CSVs from Rosreestr/etc — too large for git)
|
||||
data/raw/
|
||||
*.csv
|
||||
*.csv.zip
|
||||
!backend/db/init/*.sql
|
||||
|
|
|
|||
92
data/sql/01_schema_rosreestr_deals.sql
Normal file
92
data/sql/01_schema_rosreestr_deals.sql
Normal file
|
|
@ -0,0 +1,92 @@
|
|||
-- Rosreestr open dataset: dataset_СДЕЛКИ (registered real estate deals).
|
||||
-- Source: https://rosreestr.gov.ru/data-sets/{Q}%20квартал%20{YYYY}г./dataset_СДЕЛКИ_r-r_01-92_y_{YYYY}_q_{Q}.csv.zip
|
||||
-- Encoding: UTF-8. Separator: '~' (since 2024 Q4) or ';' (2024 Q3 only).
|
||||
-- Granularity: aggregated rows — same (region+district+city+cad_quarter+street+type+material+year+floor+purpose+area+doc_type) deals
|
||||
-- collapsed into a single row with `deal_count` (was `number`) and AVERAGE price.
|
||||
-- Time granularity: quarter (period_start_date = first day of quarter).
|
||||
-- Coverage: ALL registered transactions (DKP=secondary+land+commercial, DDU=primary in МКД).
|
||||
-- Excluded: gratuitous deals (gift/inheritance), buyer/seller PII, exact deal date, apartment number, room count, mortgage flag.
|
||||
-- Data quality notes (observed in 7 loaded quarters):
|
||||
-- * deal_price может быть мусорным с десятками знаков — фильтруем при INSERT
|
||||
-- * year_build бывает с датой типа '09.07.1988' — оставляем NULL если не 4-цифровой год
|
||||
-- * wall_material_code/realestate_type_code иногда '"code1;code2"' — поэтому TEXT вместо VARCHAR(15)
|
||||
|
||||
CREATE TABLE IF NOT EXISTS rosreestr_deals (
|
||||
id BIGSERIAL,
|
||||
-- Origin
|
||||
source_quarter VARCHAR(8) NOT NULL, -- '2024Q3' for traceability
|
||||
period_start_date DATE NOT NULL, -- first day of the quarter (partition key)
|
||||
-- Geo
|
||||
region_code SMALLINT NOT NULL, -- 66 = Свердловская обл, 77 = Москва, ...
|
||||
okato TEXT, -- ОКАТО (often '-' or 11 digits)
|
||||
district TEXT, -- муниципальный район/округ
|
||||
city TEXT, -- населённый пункт (часто пустой)
|
||||
quarter_cad_number VARCHAR(30) NOT NULL, -- '66:41:0506001' — гео-ключ для JOIN с ПКК / NSPD
|
||||
street TEXT,
|
||||
-- Object
|
||||
realestate_type_code TEXT, -- 002001001000=ЗУ | 002001002000=Здание | 002001003000=Помещение | 002001009000=Машиноместо
|
||||
wall_material_code TEXT, -- справочник 126_УНСИ (может быть multi-value)
|
||||
purpose_code TEXT, -- 66/67/148_УНСИ
|
||||
year_build SMALLINT,
|
||||
floor TEXT,
|
||||
area NUMERIC, -- м²
|
||||
-- Deal
|
||||
doc_type VARCHAR(8) NOT NULL, -- 'ДКП' | 'ДДУ'
|
||||
deal_price NUMERIC, -- средняя если deal_count > 1
|
||||
currency VARCHAR(20) DEFAULT 'рубль',
|
||||
deal_count INT NOT NULL DEFAULT 1, -- было `number`
|
||||
-- Computed
|
||||
price_per_sqm NUMERIC GENERATED ALWAYS AS (
|
||||
CASE WHEN area > 0 AND deal_price > 0 THEN deal_price / area ELSE NULL END
|
||||
) STORED,
|
||||
-- Audit
|
||||
loaded_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (id, period_start_date)
|
||||
) PARTITION BY RANGE (period_start_date);
|
||||
|
||||
-- Quarterly partitions for available data (2024 Q3 — 2026 Q1)
|
||||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2024q3 PARTITION OF rosreestr_deals
|
||||
FOR VALUES FROM ('2024-07-01') TO ('2024-10-01');
|
||||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2024q4 PARTITION OF rosreestr_deals
|
||||
FOR VALUES FROM ('2024-10-01') TO ('2025-01-01');
|
||||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q1 PARTITION OF rosreestr_deals
|
||||
FOR VALUES FROM ('2025-01-01') TO ('2025-04-01');
|
||||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q2 PARTITION OF rosreestr_deals
|
||||
FOR VALUES FROM ('2025-04-01') TO ('2025-07-01');
|
||||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q3 PARTITION OF rosreestr_deals
|
||||
FOR VALUES FROM ('2025-07-01') TO ('2025-10-01');
|
||||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q4 PARTITION OF rosreestr_deals
|
||||
FOR VALUES FROM ('2025-10-01') TO ('2026-01-01');
|
||||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2026q1 PARTITION OF rosreestr_deals
|
||||
FOR VALUES FROM ('2026-01-01') TO ('2026-04-01');
|
||||
|
||||
-- Indexes
|
||||
CREATE INDEX IF NOT EXISTS rosreestr_deals_region_cq_idx
|
||||
ON rosreestr_deals (region_code, quarter_cad_number);
|
||||
CREATE INDEX IF NOT EXISTS rosreestr_deals_period_brin
|
||||
ON rosreestr_deals USING BRIN (period_start_date);
|
||||
CREATE INDEX IF NOT EXISTS rosreestr_deals_dealtype_idx
|
||||
ON rosreestr_deals (doc_type) WHERE doc_type IN ('ДКП', 'ДДУ');
|
||||
CREATE INDEX IF NOT EXISTS rosreestr_deals_realtype_idx
|
||||
ON rosreestr_deals (realestate_type_code);
|
||||
|
||||
-- Staging: один общий unlogged-буфер. Перед каждым кварталом TRUNCATE.
|
||||
CREATE UNLOGGED TABLE IF NOT EXISTS rosreestr_deals_staging (
|
||||
deal_count INT,
|
||||
okato TEXT,
|
||||
region_code SMALLINT,
|
||||
district TEXT,
|
||||
city TEXT,
|
||||
quarter_cad_number TEXT,
|
||||
street TEXT,
|
||||
realestate_type_code TEXT,
|
||||
wall_material_code TEXT,
|
||||
year_build TEXT,
|
||||
floor TEXT,
|
||||
purpose_code TEXT,
|
||||
area NUMERIC,
|
||||
period_start_date DATE,
|
||||
deal_price NUMERIC,
|
||||
currency VARCHAR(20),
|
||||
doc_type VARCHAR(8)
|
||||
);
|
||||
96
data/sql/02_load_all_quarters.sh
Normal file
96
data/sql/02_load_all_quarters.sh
Normal file
|
|
@ -0,0 +1,96 @@
|
|||
#!/usr/bin/env bash
|
||||
# Loads all 7 quarters of dataset_СДЕЛКИ into rosreestr_deals via staging.
|
||||
# Q3 2024 uses ';' separator, all later quarters use '~'.
|
||||
#
|
||||
# Usage:
|
||||
# bash 02_load_all_quarters.sh # local test container (gd_test_pg)
|
||||
# PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... bash 02_load_all_quarters.sh # remote (via SSH tunnel)
|
||||
set -euo pipefail
|
||||
|
||||
if [[ -n "${PG_HOST:-}" ]]; then
|
||||
# Remote / TCP mode (e.g. prod via SSH tunnel)
|
||||
PG_PORT="${PG_PORT:-5432}"
|
||||
PG_USER="${PG_USER:-gendesign}"
|
||||
PG_DB="${PG_DB:-gendesign}"
|
||||
PSQL="docker run --rm -i -e PGPASSWORD postgres:16-alpine psql -h host.docker.internal -p $PG_PORT -U $PG_USER -d $PG_DB -v ON_ERROR_STOP=1"
|
||||
PSQL_PIPE="docker run --rm -i -e PGPASSWORD postgres:16-alpine psql -h host.docker.internal -p $PG_PORT -U $PG_USER -d $PG_DB -v ON_ERROR_STOP=1"
|
||||
else
|
||||
# Local docker exec into the test container
|
||||
PG_CONTAINER="${PG_CONTAINER:-gd_test_pg}"
|
||||
PSQL="docker exec -i $PG_CONTAINER psql -U gendesign -d gendesign -v ON_ERROR_STOP=1"
|
||||
PSQL_PIPE="$PSQL"
|
||||
fi
|
||||
|
||||
cd "$(dirname "$0")/../raw"
|
||||
|
||||
declare -a JOBS=(
|
||||
"2024Q3:2024-07-01:dataset_СДЕЛКИ_r_all_q_3.csv:;"
|
||||
"2024Q4:2024-10-01:dataset_СДЕЛКИ_r-r_01-92_y_2024_q_4.csv:~"
|
||||
"2025Q1:2025-01-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_1.csv:~"
|
||||
"2025Q2:2025-04-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_2.csv:~"
|
||||
"2025Q3:2025-07-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_3.csv:~"
|
||||
"2025Q4:2025-10-01:dataset_СДЕЛКИ_r-r_01-92_y_2025_q_4.csv:~"
|
||||
"2026Q1:2026-01-01:dataset_СДЕЛКИ_r-r_01-92_y_2026_q_1.csv:~"
|
||||
)
|
||||
|
||||
for job in "${JOBS[@]}"; do
|
||||
IFS=':' read -r SRC_Q PERIOD FILE SEP <<< "$job"
|
||||
echo "=== $SRC_Q ($FILE, sep='$SEP') ==="
|
||||
$PSQL -c "TRUNCATE rosreestr_deals_staging;"
|
||||
start=$(date +%s)
|
||||
cat "$FILE" | $PSQL -c "\\copy rosreestr_deals_staging FROM stdin (FORMAT csv, HEADER true, DELIMITER '$SEP', QUOTE '\"', NULL '')"
|
||||
$PSQL <<SQL
|
||||
INSERT INTO rosreestr_deals (
|
||||
source_quarter, period_start_date,
|
||||
region_code, okato, district, city, quarter_cad_number, street,
|
||||
realestate_type_code, wall_material_code, purpose_code,
|
||||
year_build, floor, area,
|
||||
doc_type, deal_price, currency, deal_count
|
||||
)
|
||||
SELECT
|
||||
'$SRC_Q', period_start_date,
|
||||
region_code, NULLIF(okato, '-'), NULLIF(district, ''), NULLIF(city, ''),
|
||||
quarter_cad_number, NULLIF(street, ''),
|
||||
NULLIF(realestate_type_code, ''),
|
||||
NULLIF(wall_material_code, ''),
|
||||
NULLIF(purpose_code, ''),
|
||||
-- year_build can contain garbage like '09.07.1988' — keep only plausible 4-digit years
|
||||
CASE WHEN year_build ~ '^[12][0-9]{3}$' THEN year_build::INT ELSE NULL END,
|
||||
NULLIF(floor, ''),
|
||||
-- discard absurd area/price (data-quality outliers from source)
|
||||
CASE WHEN area > 0 AND area < 1e8 THEN area ELSE NULL END,
|
||||
NULLIF(doc_type, ''),
|
||||
CASE WHEN deal_price > 0 AND deal_price < 1e15 THEN deal_price ELSE NULL END,
|
||||
COALESCE(NULLIF(currency, ''), 'рубль'),
|
||||
COALESCE(deal_count, 1)
|
||||
FROM rosreestr_deals_staging
|
||||
WHERE region_code IS NOT NULL
|
||||
AND quarter_cad_number IS NOT NULL
|
||||
AND quarter_cad_number <> ''
|
||||
AND doc_type IS NOT NULL
|
||||
AND doc_type <> ''
|
||||
AND period_start_date IS NOT NULL;
|
||||
SQL
|
||||
end=$(date +%s)
|
||||
echo " loaded in $((end-start))s"
|
||||
done
|
||||
|
||||
$PSQL <<'SQL'
|
||||
SELECT
|
||||
source_quarter,
|
||||
COUNT(*) AS rows_loaded,
|
||||
SUM(deal_count) AS deals,
|
||||
SUM(deal_count) FILTER (WHERE region_code = 66) AS sverdl_deals,
|
||||
pg_size_pretty(pg_relation_size('rosreestr_deals_'||lower(source_quarter))) AS partition_size
|
||||
FROM rosreestr_deals
|
||||
GROUP BY source_quarter
|
||||
ORDER BY source_quarter;
|
||||
|
||||
SELECT
|
||||
'TOTAL' AS quarter,
|
||||
COUNT(*) AS rows,
|
||||
SUM(deal_count) AS deals,
|
||||
SUM(deal_count) FILTER (WHERE region_code = 66) AS sverdl_deals,
|
||||
pg_size_pretty(pg_total_relation_size('rosreestr_deals')) AS total_size
|
||||
FROM rosreestr_deals;
|
||||
SQL
|
||||
98
data/sql/03_materialized_views.sql
Normal file
98
data/sql/03_materialized_views.sql
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
-- Materialized aggregates for fast UI queries.
|
||||
-- Refresh strategy: REFRESH MATERIALIZED VIEW CONCURRENTLY after each new quarter load.
|
||||
|
||||
-- 1. Per cadastral quarter × period × doc_type × area_bucket
|
||||
DROP MATERIALIZED VIEW IF EXISTS rr_agg_cad_quarter CASCADE;
|
||||
CREATE MATERIALIZED VIEW rr_agg_cad_quarter AS
|
||||
SELECT
|
||||
region_code,
|
||||
quarter_cad_number,
|
||||
period_start_date,
|
||||
doc_type,
|
||||
-- area bucket inspired by NSPD MarketAnalytics filter
|
||||
CASE
|
||||
WHEN area < 25 THEN 'lt25'
|
||||
WHEN area < 40 THEN '25-40'
|
||||
WHEN area < 60 THEN '40-60'
|
||||
WHEN area < 90 THEN '60-90'
|
||||
WHEN area < 150 THEN '90-150'
|
||||
ELSE 'gt150'
|
||||
END AS area_bucket,
|
||||
SUM(deal_count) AS deals_count,
|
||||
COUNT(*) AS rows_count,
|
||||
SUM(deal_count * area) AS total_area_sqm,
|
||||
SUM(deal_count * deal_price) AS total_value_rub,
|
||||
percentile_cont(0.10) WITHIN GROUP (ORDER BY price_per_sqm) AS p10_price_sqm,
|
||||
percentile_cont(0.50) WITHIN GROUP (ORDER BY price_per_sqm) AS median_price_sqm,
|
||||
percentile_cont(0.90) WITHIN GROUP (ORDER BY price_per_sqm) AS p90_price_sqm,
|
||||
AVG(price_per_sqm) AS mean_price_sqm,
|
||||
MIN(price_per_sqm) AS min_price_sqm,
|
||||
MAX(price_per_sqm) AS max_price_sqm
|
||||
FROM rosreestr_deals
|
||||
WHERE area BETWEEN 5 AND 1000
|
||||
AND deal_price BETWEEN 100000 AND 1e10 -- discard outliers (typos, junk in source)
|
||||
AND realestate_type_code IN ('002001003000') -- помещения (квартиры). Здания/ЗУ — отдельный rollup
|
||||
GROUP BY region_code, quarter_cad_number, period_start_date, doc_type,
|
||||
CASE WHEN area<25 THEN 'lt25' WHEN area<40 THEN '25-40' WHEN area<60 THEN '40-60'
|
||||
WHEN area<90 THEN '60-90' WHEN area<150 THEN '90-150' ELSE 'gt150' END
|
||||
WITH NO DATA;
|
||||
|
||||
CREATE UNIQUE INDEX rr_agg_cq_pk ON rr_agg_cad_quarter
|
||||
(region_code, quarter_cad_number, period_start_date, doc_type, area_bucket);
|
||||
CREATE INDEX rr_agg_cq_period_idx ON rr_agg_cad_quarter (period_start_date);
|
||||
CREATE INDEX rr_agg_cq_region_period_idx ON rr_agg_cad_quarter (region_code, period_start_date);
|
||||
|
||||
-- 2. Per region × period × doc_type — для регионального обзора (Урал dashboard)
|
||||
DROP MATERIALIZED VIEW IF EXISTS rr_agg_region CASCADE;
|
||||
CREATE MATERIALIZED VIEW rr_agg_region AS
|
||||
SELECT
|
||||
region_code,
|
||||
period_start_date,
|
||||
doc_type,
|
||||
realestate_type_code,
|
||||
SUM(deal_count) AS deals_count,
|
||||
COUNT(*) AS rows_count,
|
||||
SUM(deal_count * area) AS total_area_sqm,
|
||||
SUM(deal_count * deal_price) AS total_value_rub,
|
||||
percentile_cont(0.10) WITHIN GROUP (ORDER BY price_per_sqm) AS p10_price_sqm,
|
||||
percentile_cont(0.50) WITHIN GROUP (ORDER BY price_per_sqm) AS median_price_sqm,
|
||||
percentile_cont(0.90) WITHIN GROUP (ORDER BY price_per_sqm) AS p90_price_sqm
|
||||
FROM rosreestr_deals
|
||||
WHERE area BETWEEN 5 AND 1000
|
||||
AND deal_price BETWEEN 100000 AND 1e10
|
||||
GROUP BY region_code, period_start_date, doc_type, realestate_type_code
|
||||
WITH NO DATA;
|
||||
|
||||
CREATE UNIQUE INDEX rr_agg_region_pk ON rr_agg_region
|
||||
(region_code, period_start_date, doc_type, realestate_type_code);
|
||||
|
||||
-- 3. Per city/settlement (для дашборда Екатеринбург)
|
||||
DROP MATERIALIZED VIEW IF EXISTS rr_agg_settlement CASCADE;
|
||||
CREATE MATERIALIZED VIEW rr_agg_settlement AS
|
||||
SELECT
|
||||
region_code,
|
||||
COALESCE(city, district) AS settlement_name,
|
||||
period_start_date,
|
||||
doc_type,
|
||||
realestate_type_code,
|
||||
SUM(deal_count) AS deals_count,
|
||||
SUM(deal_count * area) AS total_area_sqm,
|
||||
percentile_cont(0.50) WITHIN GROUP (ORDER BY price_per_sqm) AS median_price_sqm,
|
||||
percentile_cont(0.10) WITHIN GROUP (ORDER BY price_per_sqm) AS p10_price_sqm,
|
||||
percentile_cont(0.90) WITHIN GROUP (ORDER BY price_per_sqm) AS p90_price_sqm,
|
||||
COUNT(DISTINCT quarter_cad_number) AS distinct_cad_quarters
|
||||
FROM rosreestr_deals
|
||||
WHERE area BETWEEN 5 AND 1000
|
||||
AND deal_price BETWEEN 100000 AND 1e10
|
||||
AND COALESCE(city, district) IS NOT NULL
|
||||
GROUP BY region_code, COALESCE(city, district), period_start_date, doc_type, realestate_type_code
|
||||
WITH NO DATA;
|
||||
|
||||
CREATE UNIQUE INDEX rr_agg_settl_pk ON rr_agg_settlement
|
||||
(region_code, settlement_name, period_start_date, doc_type, realestate_type_code);
|
||||
CREATE INDEX rr_agg_settl_search ON rr_agg_settlement (region_code, settlement_name);
|
||||
|
||||
-- Initial fill
|
||||
REFRESH MATERIALIZED VIEW rr_agg_cad_quarter;
|
||||
REFRESH MATERIALIZED VIEW rr_agg_region;
|
||||
REFRESH MATERIALIZED VIEW rr_agg_settlement;
|
||||
67
data/sql/10_schema_domrf.sql
Normal file
67
data/sql/10_schema_domrf.sql
Normal file
|
|
@ -0,0 +1,67 @@
|
|||
-- DOM.RF (наш.дом.рф) portal-analytics dashboard data.
|
||||
-- Source: https://xn--80az8a.xn--d1aqf.xn--p1ai/portal-analytics/api/*
|
||||
-- Coverage: aggregate of construction-stage flats in МКД (multi-apartment buildings).
|
||||
-- ⚠ NOT historical: this is a SNAPSHOT of "currently being built" state. Updated continuously.
|
||||
-- We store snapshot_date so we can rebuild trend over time by re-pulling.
|
||||
|
||||
-- 1. Federal districts + regions dictionary (from /dictionaries/regions)
|
||||
CREATE TABLE IF NOT EXISTS domrf_regions (
|
||||
region_id INT PRIMARY KEY,
|
||||
region_name TEXT NOT NULL,
|
||||
federal_district TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- 2. Developers dictionary (from /dictionaries/developers, 2734 entries as of 2026-04-26)
|
||||
CREATE TABLE IF NOT EXISTS domrf_developers (
|
||||
developer_id TEXT PRIMARY KEY, -- '6208_0' (PRINZIP), '5655_0' (ПИК), '0_14583' etc.
|
||||
developer_name TEXT NOT NULL
|
||||
);
|
||||
|
||||
-- 3. Snapshot metadata
|
||||
CREATE TABLE IF NOT EXISTS domrf_snapshots (
|
||||
snapshot_date DATE PRIMARY KEY,
|
||||
fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
notes TEXT
|
||||
);
|
||||
|
||||
-- 4. By-region aggregates with breakdown by room count
|
||||
-- One row per (region, room_count_type, snapshot)
|
||||
CREATE TABLE IF NOT EXISTS domrf_region_aggregates (
|
||||
region_id INT NOT NULL,
|
||||
snapshot_date DATE NOT NULL,
|
||||
room_count_type TEXT NOT NULL, -- 'ONE'|'TWO'|'THREE'|'FOUR' or 'TOTAL' for region totals
|
||||
flat_count INT NOT NULL,
|
||||
area_sqm NUMERIC(14, 2) NOT NULL,
|
||||
percent SMALLINT, -- % within region for room_count breakdown
|
||||
PRIMARY KEY (region_id, snapshot_date, room_count_type)
|
||||
);
|
||||
|
||||
-- 5. By-flat-area distribution (RF-wide, не работает фильтр по региону)
|
||||
CREATE TABLE IF NOT EXISTS domrf_flat_area_distribution (
|
||||
snapshot_date DATE NOT NULL,
|
||||
region_id INT NOT NULL DEFAULT 0, -- 0 = вся РФ (api ignores ?regionId on this page)
|
||||
area_bucket TEXT NOT NULL, -- 'FROM_0_TO_25', 'FROM_25_TO_35', ..., 'FROM_100'
|
||||
room_count_type TEXT NOT NULL, -- 'ONE'|'TWO'|'THREE'|'FOUR' or 'TOTAL'
|
||||
flat_count INT NOT NULL,
|
||||
area_sqm NUMERIC(14, 2) NOT NULL,
|
||||
percent SMALLINT,
|
||||
PRIMARY KEY (snapshot_date, region_id, area_bucket, room_count_type)
|
||||
);
|
||||
|
||||
-- 6. By-developer aggregates with breakdown by room count
|
||||
-- Topology: developer × snapshot × room_count_type
|
||||
CREATE TABLE IF NOT EXISTS domrf_developer_aggregates (
|
||||
developer_id TEXT NOT NULL,
|
||||
snapshot_date DATE NOT NULL,
|
||||
region_id INT NOT NULL DEFAULT 0, -- 0 = РФ агрегат (api ignores ?regionId on this page)
|
||||
room_count_type TEXT NOT NULL,
|
||||
flat_count INT NOT NULL,
|
||||
area_sqm NUMERIC(14, 2) NOT NULL,
|
||||
percent SMALLINT,
|
||||
PRIMARY KEY (developer_id, snapshot_date, region_id, room_count_type)
|
||||
);
|
||||
|
||||
-- Indexes
|
||||
CREATE INDEX IF NOT EXISTS idx_domrf_region_aggregates_room ON domrf_region_aggregates (room_count_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_domrf_developer_aggregates_room ON domrf_developer_aggregates (room_count_type);
|
||||
CREATE INDEX IF NOT EXISTS idx_domrf_developer_aggregates_dev ON domrf_developer_aggregates (developer_id);
|
||||
183
data/sql/11_load_domrf.py
Normal file
183
data/sql/11_load_domrf.py
Normal file
|
|
@ -0,0 +1,183 @@
|
|||
"""Loads DOM.RF analytics JSON files from data/raw/domrf/ into Postgres.
|
||||
|
||||
Usage:
|
||||
PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... python data/sql/11_load_domrf.py
|
||||
# or for local test:
|
||||
PGPASSWORD=test PG_PORT=6543 python data/sql/11_load_domrf.py
|
||||
|
||||
Files expected (snapshot from https://наш.дом.рф/portal-analytics/api/):
|
||||
data/raw/domrf/regions.json — list of FO with regions inside
|
||||
data/raw/domrf/developers.json — flat list of {id, name}
|
||||
data/raw/domrf/by-region.json — per-region aggregates with room-count details
|
||||
data/raw/domrf/by-flat-area-RF.json — RF area distribution with room-count details
|
||||
data/raw/domrf/by-developer-RF.json — RF top developers with room-count details
|
||||
data/raw/domrf/by-room-count-RF.json — RF totals (sanity check, not stored)
|
||||
"""
|
||||
import json, os, sys, datetime, subprocess
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
RAW = os.path.join(HERE, '..', 'raw', 'domrf')
|
||||
|
||||
PG_HOST = os.environ.get('PG_HOST', '127.0.0.1')
|
||||
PG_PORT = os.environ.get('PG_PORT', '5432')
|
||||
PG_USER = os.environ.get('PG_USER', 'gendesign')
|
||||
PG_DB = os.environ.get('PG_DB', 'gendesign')
|
||||
PG_PASS = os.environ.get('PGPASSWORD', '')
|
||||
|
||||
SNAPSHOT_DATE = os.environ.get('SNAPSHOT_DATE', datetime.date.today().isoformat())
|
||||
|
||||
|
||||
def psql(sql_or_stdin, stdin_text=None):
|
||||
"""Run psql via docker. Pipe SQL via stdin."""
|
||||
cmd = [
|
||||
'docker', 'run', '--rm', '-i',
|
||||
'-e', f'PGPASSWORD={PG_PASS}',
|
||||
'postgres:16-alpine',
|
||||
'psql', '-h', 'host.docker.internal', '-p', PG_PORT, '-U', PG_USER, '-d', PG_DB,
|
||||
'-v', 'ON_ERROR_STOP=1', '--quiet',
|
||||
]
|
||||
if stdin_text is None:
|
||||
stdin_text = sql_or_stdin
|
||||
res = subprocess.run(cmd, input=stdin_text, capture_output=True, text=True, encoding='utf-8')
|
||||
if res.returncode != 0:
|
||||
print('psql FAILED:', res.stderr, file=sys.stderr)
|
||||
raise SystemExit(res.returncode)
|
||||
return res.stdout
|
||||
|
||||
|
||||
def sql_escape(s):
|
||||
return s.replace("'", "''")
|
||||
|
||||
|
||||
def main():
|
||||
raw_files = {
|
||||
'regions': os.path.join(RAW, 'regions.json'),
|
||||
'developers': os.path.join(RAW, 'developers.json'),
|
||||
'by-region': os.path.join(RAW, 'by-region.json'),
|
||||
'by-flat-area-RF': os.path.join(RAW, 'by-flat-area-RF.json'),
|
||||
'by-developer-RF': os.path.join(RAW, 'by-developer-RF.json'),
|
||||
}
|
||||
for k, p in raw_files.items():
|
||||
if not os.path.exists(p):
|
||||
print(f'MISSING: {p}', file=sys.stderr); raise SystemExit(2)
|
||||
|
||||
# 0. Apply schema (idempotent CREATE TABLE IF NOT EXISTS)
|
||||
schema = open(os.path.join(HERE, '10_schema_domrf.sql'), encoding='utf-8').read()
|
||||
psql(schema)
|
||||
print('schema applied')
|
||||
|
||||
# 1. Snapshot row
|
||||
psql(f"""
|
||||
INSERT INTO domrf_snapshots (snapshot_date, notes)
|
||||
VALUES ('{SNAPSHOT_DATE}', 'naash.dom.rf portal-analytics, captured via chrome-devtools MCP')
|
||||
ON CONFLICT (snapshot_date) DO UPDATE SET fetched_at = NOW();
|
||||
""")
|
||||
|
||||
# 2. Regions dictionary
|
||||
regions_data = json.load(open(raw_files['regions'], encoding='utf-8'))
|
||||
parts = []
|
||||
for fo in regions_data:
|
||||
for r in fo['regions']:
|
||||
parts.append(f"({r['id']}, '{sql_escape(r['name'])}', '{sql_escape(fo['name'])}')")
|
||||
if parts:
|
||||
psql(f"""
|
||||
INSERT INTO domrf_regions (region_id, region_name, federal_district)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (region_id) DO UPDATE SET
|
||||
region_name = EXCLUDED.region_name,
|
||||
federal_district = EXCLUDED.federal_district;
|
||||
""")
|
||||
print(f'regions: {sum(len(fo["regions"]) for fo in regions_data)}')
|
||||
|
||||
# 3. Developers dictionary
|
||||
devs = json.load(open(raw_files['developers'], encoding='utf-8'))
|
||||
# batch insert in chunks of 500 to keep statement size sane
|
||||
BATCH = 500
|
||||
for i in range(0, len(devs), BATCH):
|
||||
chunk = devs[i:i+BATCH]
|
||||
parts = [f"('{sql_escape(d['id'])}', '{sql_escape(d['name'])}')" for d in chunk]
|
||||
psql(f"""
|
||||
INSERT INTO domrf_developers (developer_id, developer_name)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (developer_id) DO UPDATE SET developer_name = EXCLUDED.developer_name;
|
||||
""")
|
||||
print(f'developers: {len(devs)}')
|
||||
|
||||
# 4. By-region aggregates: insert TOTAL row + per-room rows for each region
|
||||
byregion = json.load(open(raw_files['by-region'], encoding='utf-8'))
|
||||
rows = []
|
||||
for r in byregion:
|
||||
rid = r['id']
|
||||
# TOTAL row
|
||||
rows.append((rid, SNAPSHOT_DATE, 'TOTAL', r['flatCount'], r['area'], None))
|
||||
for d in r.get('details', []):
|
||||
rows.append((rid, SNAPSHOT_DATE, d['roomCountType'], d['flatCount'], d['area'], d.get('percent')))
|
||||
parts = [f"({rid}, '{sd}', '{rt}', {fc}, {a}, {'NULL' if p is None else p})"
|
||||
for (rid, sd, rt, fc, a, p) in rows]
|
||||
for i in range(0, len(parts), BATCH):
|
||||
psql(f"""
|
||||
INSERT INTO domrf_region_aggregates (region_id, snapshot_date, room_count_type, flat_count, area_sqm, percent)
|
||||
VALUES {','.join(parts[i:i+BATCH])}
|
||||
ON CONFLICT (region_id, snapshot_date, room_count_type) DO UPDATE SET
|
||||
flat_count = EXCLUDED.flat_count,
|
||||
area_sqm = EXCLUDED.area_sqm,
|
||||
percent = EXCLUDED.percent;
|
||||
""")
|
||||
print(f'region_aggregates rows: {len(rows)}')
|
||||
|
||||
# 5. Flat-area distribution (RF-wide)
|
||||
byarea = json.load(open(raw_files['by-flat-area-RF'], encoding='utf-8'))
|
||||
rows = []
|
||||
for bucket in byarea:
|
||||
bid = bucket['id']
|
||||
rows.append((bid, 'TOTAL', bucket['flatCount'], bucket['area'], bucket.get('percent')))
|
||||
for d in bucket.get('details', []):
|
||||
rows.append((bid, d['roomCountType'], d['flatCount'], d['area'], d.get('percent')))
|
||||
parts = [f"('{SNAPSHOT_DATE}', 0, '{sql_escape(bid)}', '{rt}', {fc}, {a}, {'NULL' if p is None else p})"
|
||||
for (bid, rt, fc, a, p) in rows]
|
||||
psql(f"""
|
||||
INSERT INTO domrf_flat_area_distribution (snapshot_date, region_id, area_bucket, room_count_type, flat_count, area_sqm, percent)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, region_id, area_bucket, room_count_type) DO UPDATE SET
|
||||
flat_count = EXCLUDED.flat_count,
|
||||
area_sqm = EXCLUDED.area_sqm,
|
||||
percent = EXCLUDED.percent;
|
||||
""")
|
||||
print(f'flat_area_distribution rows: {len(rows)}')
|
||||
|
||||
# 6. By-developer aggregates (RF-wide because ?regionId is ignored)
|
||||
bydev = json.load(open(raw_files['by-developer-RF'], encoding='utf-8'))
|
||||
rows = []
|
||||
for dev in bydev:
|
||||
did = dev['id']
|
||||
rows.append((did, 'TOTAL', dev['flatCount'], dev['area'], dev.get('percent')))
|
||||
for d in dev.get('details', []):
|
||||
rows.append((did, d['roomCountType'], d['flatCount'], d['area'], d.get('percent')))
|
||||
parts = [f"('{sql_escape(did)}', '{SNAPSHOT_DATE}', 0, '{rt}', {fc}, {a}, {'NULL' if p is None else p})"
|
||||
for (did, rt, fc, a, p) in rows]
|
||||
for i in range(0, len(parts), BATCH):
|
||||
psql(f"""
|
||||
INSERT INTO domrf_developer_aggregates (developer_id, snapshot_date, region_id, room_count_type, flat_count, area_sqm, percent)
|
||||
VALUES {','.join(parts[i:i+BATCH])}
|
||||
ON CONFLICT (developer_id, snapshot_date, region_id, room_count_type) DO UPDATE SET
|
||||
flat_count = EXCLUDED.flat_count,
|
||||
area_sqm = EXCLUDED.area_sqm,
|
||||
percent = EXCLUDED.percent;
|
||||
""")
|
||||
print(f'developer_aggregates rows: {len(rows)}')
|
||||
|
||||
# Final summary
|
||||
out = psql("""
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM domrf_regions) AS regions,
|
||||
(SELECT COUNT(*) FROM domrf_developers) AS developers,
|
||||
(SELECT COUNT(*) FROM domrf_region_aggregates) AS region_agg,
|
||||
(SELECT COUNT(*) FROM domrf_flat_area_distribution) AS area_dist,
|
||||
(SELECT COUNT(*) FROM domrf_developer_aggregates) AS dev_agg;
|
||||
""")
|
||||
print('--- summary ---')
|
||||
print(out)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
136
data/sql/20_schema_domrf_extras.sql
Normal file
136
data/sql/20_schema_domrf_extras.sql
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
-- DOM.RF additional snapshots (downloaded as XLSX/PDF from наш.дом.рф via "Скачать" buttons).
|
||||
-- Snapshot date stamped on the file: 26.04.2026 (mortgage_rates is 19.04.2026).
|
||||
-- These extend the JSON dashboard data with: full developer registry, escrow banks,
|
||||
-- 214-FZ guaranty per region, sold-out (Tile3 velocity), housing info by mechanism, mortgage rates.
|
||||
|
||||
-- 1. Полный реестр застройщиков с метриками (НЕ группа компаний — а юрлица из проектных деклараций)
|
||||
CREATE TABLE IF NOT EXISTS domrf_developers_full (
|
||||
snapshot_date DATE NOT NULL,
|
||||
developer_name TEXT NOT NULL,
|
||||
area_thousand_sqm NUMERIC,
|
||||
permits_count INT,
|
||||
houses_count INT,
|
||||
flats_count INT,
|
||||
market_share_pct NUMERIC,
|
||||
PRIMARY KEY (snapshot_date, developer_name)
|
||||
);
|
||||
|
||||
-- 2. Банки эскроу с метриками
|
||||
CREATE TABLE IF NOT EXISTS domrf_escrow_banks (
|
||||
snapshot_date DATE NOT NULL,
|
||||
bank_name TEXT NOT NULL,
|
||||
houses_count INT,
|
||||
permits_count INT,
|
||||
area_thousand_sqm NUMERIC,
|
||||
developers_count INT,
|
||||
regions_count INT,
|
||||
PRIMARY KEY (snapshot_date, bank_name)
|
||||
);
|
||||
|
||||
-- 3. Долевое стр-во по 214-ФЗ — по регионам с разбивкой эскроу/ПП480
|
||||
CREATE TABLE IF NOT EXISTS domrf_guaranty_regions (
|
||||
snapshot_date DATE NOT NULL,
|
||||
territory_name TEXT NOT NULL, -- 'Российская Федерация' | 'Центральный ФО' | 'Свердловская область' ...
|
||||
territory_level TEXT NOT NULL, -- 'rf' | 'fo' | 'region'
|
||||
-- Всего строится
|
||||
total_permits INT,
|
||||
total_area_th_sqm NUMERIC,
|
||||
total_developers INT,
|
||||
-- Имеют право привлекать ДУ по 214-ФЗ (всего)
|
||||
fz214_permits INT,
|
||||
fz214_area_th_sqm NUMERIC,
|
||||
fz214_developers INT,
|
||||
fz214_pct_of_total NUMERIC,
|
||||
-- В т.ч. с эскроу
|
||||
escrow_permits INT,
|
||||
escrow_area_th_sqm NUMERIC,
|
||||
escrow_developers INT,
|
||||
-- В т.ч. ПП №480 (без эскроу)
|
||||
pp480_permits INT,
|
||||
pp480_area_th_sqm NUMERIC,
|
||||
pp480_developers INT,
|
||||
PRIMARY KEY (snapshot_date, territory_name)
|
||||
);
|
||||
|
||||
-- 4. Сводные показатели по механизмам привлечения (housing_info)
|
||||
CREATE TABLE IF NOT EXISTS domrf_housing_summary (
|
||||
snapshot_date DATE NOT NULL,
|
||||
territory_name TEXT NOT NULL, -- 'Российская Федерация' | город | регион
|
||||
territory_level TEXT NOT NULL, -- 'rf' | 'region' | 'city'
|
||||
mechanism TEXT NOT NULL, -- 'all' | 'escrow' | 'fund' | 'no_attraction'
|
||||
developers_count INT,
|
||||
permits_count INT,
|
||||
declarations_count INT,
|
||||
houses_count INT,
|
||||
area_thousand_sqm NUMERIC,
|
||||
flats_thousand NUMERIC,
|
||||
PRIMARY KEY (snapshot_date, territory_name, mechanism)
|
||||
);
|
||||
|
||||
-- 5. Плановые сроки ввода (housing_info, нижняя таблица)
|
||||
CREATE TABLE IF NOT EXISTS domrf_planned_commissioning (
|
||||
snapshot_date DATE NOT NULL,
|
||||
plan_year INT NOT NULL,
|
||||
escrow_th_sqm NUMERIC,
|
||||
fund_th_sqm NUMERIC,
|
||||
no_attraction_th_sqm NUMERIC,
|
||||
total_th_sqm NUMERIC,
|
||||
PRIMARY KEY (snapshot_date, plan_year)
|
||||
);
|
||||
|
||||
-- 6. Реализация (sold_out_info) — РФ итог + регионы
|
||||
CREATE TABLE IF NOT EXISTS domrf_sold_out (
|
||||
snapshot_date DATE NOT NULL, -- '2026-04-11' (отчёт за март 2026)
|
||||
territory_name TEXT NOT NULL,
|
||||
-- Площади
|
||||
total_area_th_sqm NUMERIC,
|
||||
sold_area_th_sqm NUMERIC,
|
||||
unsold_area_th_sqm NUMERIC,
|
||||
not_open_area_th_sqm NUMERIC,
|
||||
-- Проценты
|
||||
sold_pct NUMERIC,
|
||||
unsold_pct NUMERIC,
|
||||
not_open_pct NUMERIC,
|
||||
-- Дополнительно (только для РФ)
|
||||
avg_price_per_sqm NUMERIC, -- ₽/м²
|
||||
attracted_funds_mln_rub NUMERIC, -- млн ₽
|
||||
PRIMARY KEY (snapshot_date, territory_name)
|
||||
);
|
||||
|
||||
-- 7. Реализация по году ввода
|
||||
CREATE TABLE IF NOT EXISTS domrf_sold_out_by_year (
|
||||
snapshot_date DATE NOT NULL,
|
||||
plan_year TEXT NOT NULL, -- '2026' .. '2031+'
|
||||
total_th_sqm NUMERIC,
|
||||
pct_of_total NUMERIC,
|
||||
sold_pct NUMERIC,
|
||||
unsold_pct NUMERIC,
|
||||
not_open_pct NUMERIC,
|
||||
PRIMARY KEY (snapshot_date, plan_year)
|
||||
);
|
||||
|
||||
-- 8. Распределение по % реализации в доме
|
||||
CREATE TABLE IF NOT EXISTS domrf_sold_out_by_progress (
|
||||
snapshot_date DATE NOT NULL,
|
||||
progress_bucket TEXT NOT NULL, -- 'not_open' | '0-20' | '21-40' | '41-60' | '61-80' | '80+'
|
||||
total_th_sqm NUMERIC,
|
||||
pct_of_total NUMERIC,
|
||||
sold_pct_within NUMERIC,
|
||||
unsold_pct_within NUMERIC,
|
||||
PRIMARY KEY (snapshot_date, progress_bucket)
|
||||
);
|
||||
|
||||
-- 9. Ставки ипотеки топ-20 банков (mortgage_rates)
|
||||
CREATE TABLE IF NOT EXISTS domrf_mortgage_rates (
|
||||
snapshot_date DATE NOT NULL,
|
||||
bank_name TEXT NOT NULL,
|
||||
primary_rate NUMERIC, -- %, NULL если 'не предоставляет'
|
||||
secondary_rate NUMERIC,
|
||||
refinance_rate NUMERIC,
|
||||
note TEXT, -- 'Максимальный срок – 19 лет', 'ПВ от 50%' etc.
|
||||
PRIMARY KEY (snapshot_date, bank_name)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_domrf_developers_full_share ON domrf_developers_full (market_share_pct DESC);
|
||||
CREATE INDEX IF NOT EXISTS idx_domrf_guaranty_level ON domrf_guaranty_regions (territory_level);
|
||||
CREATE INDEX IF NOT EXISTS idx_domrf_housing_summary_terr ON domrf_housing_summary (territory_name, mechanism);
|
||||
405
data/sql/21_load_domrf_extras.py
Normal file
405
data/sql/21_load_domrf_extras.py
Normal file
|
|
@ -0,0 +1,405 @@
|
|||
"""Loads DOM.RF extra snapshots (XLSX downloads + PDF-extracted data) into Postgres.
|
||||
|
||||
Inputs in C:\\Users\\user\\Downloads\\:
|
||||
- developer_rf_26-04-2026.xlsx → domrf_developers_full
|
||||
- escrow_rf_26-04-2026.xlsx → domrf_escrow_banks
|
||||
- guaranty_26-04-2026.xlsx → domrf_guaranty_regions
|
||||
- housing_info_26-04-2026.pdf → domrf_housing_summary, domrf_planned_commissioning (hardcoded extract)
|
||||
- sold_out_info_26-04-2026.pdf → domrf_sold_out, ..._by_year, ..._by_progress (hardcoded extract)
|
||||
- mortgage_rates_19-04-2026.pdf → domrf_mortgage_rates (hardcoded extract)
|
||||
|
||||
Usage:
|
||||
PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... python data/sql/21_load_domrf_extras.py
|
||||
"""
|
||||
import os, sys, subprocess
|
||||
import openpyxl
|
||||
|
||||
DOWNLOADS = r'C:\Users\user\Downloads'
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
PG_PORT = os.environ.get('PG_PORT', '5432')
|
||||
PG_USER = os.environ.get('PG_USER', 'gendesign')
|
||||
PG_DB = os.environ.get('PG_DB', 'gendesign')
|
||||
PG_PASS = os.environ.get('PGPASSWORD', '')
|
||||
|
||||
|
||||
def psql(sql):
|
||||
cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
|
||||
'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT,
|
||||
'-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet']
|
||||
res = subprocess.run(cmd, input=sql, capture_output=True, text=True, encoding='utf-8')
|
||||
if res.returncode != 0:
|
||||
print('psql FAILED:', res.stderr[-2000:], file=sys.stderr)
|
||||
raise SystemExit(res.returncode)
|
||||
return res.stdout
|
||||
|
||||
|
||||
def esc(s):
|
||||
if s is None: return 'NULL'
|
||||
return "'" + str(s).replace("'", "''") + "'"
|
||||
|
||||
def num(v):
|
||||
if v is None or v == '-' or v == '': return 'NULL'
|
||||
try: return str(float(v))
|
||||
except (ValueError, TypeError): return 'NULL'
|
||||
|
||||
def integer(v):
|
||||
if v is None or v == '-' or v == '': return 'NULL'
|
||||
try: return str(int(float(v)))
|
||||
except (ValueError, TypeError): return 'NULL'
|
||||
|
||||
|
||||
def main():
|
||||
# 0. Apply schema
|
||||
schema = open(os.path.join(HERE, '20_schema_domrf_extras.sql'), encoding='utf-8').read()
|
||||
psql(schema)
|
||||
print('schema applied')
|
||||
|
||||
SNAP_APR26 = '2026-04-26'
|
||||
SNAP_MORTGAGE = '2026-04-19'
|
||||
SNAP_SOLD = '2026-04-11' # отчётная дата на самом PDF
|
||||
|
||||
# ─────────── 1. developer_rf.xlsx → domrf_developers_full ───────────
|
||||
wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'developer_rf_26-04-2026.xlsx'),
|
||||
read_only=True, data_only=True)
|
||||
rows = list(wb.active.iter_rows(values_only=True))
|
||||
wb.close()
|
||||
# rows[3] = header; rows[4] = numeric column index; rows[5] = 'Итого'; rows[6+] = data
|
||||
seen = {}
|
||||
for r in rows[6:]:
|
||||
name = r[0]
|
||||
if not name or not isinstance(name, str) or name.strip() == '': continue
|
||||
# Deduplicate: keep entry with highest area (xlsx может содержать дубли названий)
|
||||
key = name.strip()
|
||||
prev = seen.get(key)
|
||||
cur_area = r[1] if r[1] is not None else 0
|
||||
if prev is None or (prev[1] or 0) < cur_area:
|
||||
seen[key] = r
|
||||
parts = [f"('{SNAP_APR26}', {esc(k)}, {num(r[1])}, {integer(r[2])}, "
|
||||
f"{integer(r[3])}, {integer(r[4])}, {num(r[5])})"
|
||||
for k, r in seen.items()]
|
||||
BATCH = 500
|
||||
for i in range(0, len(parts), BATCH):
|
||||
psql(f"""
|
||||
INSERT INTO domrf_developers_full (snapshot_date, developer_name, area_thousand_sqm,
|
||||
permits_count, houses_count, flats_count, market_share_pct)
|
||||
VALUES {','.join(parts[i:i+BATCH])}
|
||||
ON CONFLICT (snapshot_date, developer_name) DO UPDATE SET
|
||||
area_thousand_sqm = EXCLUDED.area_thousand_sqm,
|
||||
permits_count = EXCLUDED.permits_count,
|
||||
houses_count = EXCLUDED.houses_count,
|
||||
flats_count = EXCLUDED.flats_count,
|
||||
market_share_pct = EXCLUDED.market_share_pct;
|
||||
""")
|
||||
print(f'developers_full: {len(parts)} (deduped from {len(rows)-6})')
|
||||
|
||||
# ─────────── 2. escrow_rf.xlsx → domrf_escrow_banks ───────────
|
||||
wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'escrow_rf_26-04-2026.xlsx'),
|
||||
read_only=True, data_only=True)
|
||||
rows = list(wb.active.iter_rows(values_only=True))
|
||||
wb.close()
|
||||
parts = []
|
||||
for r in rows[6:]:
|
||||
name = r[0]
|
||||
if not name or not isinstance(name, str) or name.strip() == '' or name == 'Итого': continue
|
||||
parts.append(f"('{SNAP_APR26}', {esc(name.strip())}, {integer(r[1])}, {integer(r[2])}, "
|
||||
f"{num(r[3])}, {integer(r[4])}, {integer(r[5])})")
|
||||
psql(f"""
|
||||
INSERT INTO domrf_escrow_banks (snapshot_date, bank_name, houses_count, permits_count,
|
||||
area_thousand_sqm, developers_count, regions_count)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, bank_name) DO UPDATE SET
|
||||
houses_count = EXCLUDED.houses_count,
|
||||
permits_count = EXCLUDED.permits_count,
|
||||
area_thousand_sqm = EXCLUDED.area_thousand_sqm,
|
||||
developers_count = EXCLUDED.developers_count,
|
||||
regions_count = EXCLUDED.regions_count;
|
||||
""")
|
||||
print(f'escrow_banks: {len(parts)}')
|
||||
|
||||
# ─────────── 3. guaranty.xlsx → domrf_guaranty_regions ───────────
|
||||
wb = openpyxl.load_workbook(os.path.join(DOWNLOADS, 'guaranty_26-04-2026.xlsx'),
|
||||
read_only=True, data_only=True)
|
||||
rows = list(wb.active.iter_rows(values_only=True))
|
||||
wb.close()
|
||||
# rows[6] = РФ; rows[7..] = ФО + регионы
|
||||
FO_NAMES = {'Центральный ФО', 'Северо-Западный ФО', 'Южный ФО', 'Северо-Кавказский ФО',
|
||||
'Приволжский ФО', 'Уральский ФО', 'Сибирский ФО', 'Дальневосточный ФО',
|
||||
'Новые субъекты Российской Федерации'}
|
||||
parts = []
|
||||
for r in rows[6:]:
|
||||
name = r[0]
|
||||
if not name or not isinstance(name, str) or name.strip() == '': continue
|
||||
name = name.strip()
|
||||
if name == 'Российская Федерация': level = 'rf'
|
||||
elif name in FO_NAMES: level = 'fo'
|
||||
else: level = 'region'
|
||||
parts.append(f"('{SNAP_APR26}', {esc(name)}, '{level}', "
|
||||
f"{integer(r[1])}, {num(r[2])}, {integer(r[3])}, "
|
||||
f"{integer(r[4])}, {num(r[5])}, {integer(r[6])}, {num(r[7])}, "
|
||||
f"{integer(r[8])}, {num(r[9])}, {integer(r[10])}, "
|
||||
f"{integer(r[11])}, {num(r[12])}, {integer(r[13])})")
|
||||
BATCH = 100
|
||||
for i in range(0, len(parts), BATCH):
|
||||
psql(f"""
|
||||
INSERT INTO domrf_guaranty_regions (snapshot_date, territory_name, territory_level,
|
||||
total_permits, total_area_th_sqm, total_developers,
|
||||
fz214_permits, fz214_area_th_sqm, fz214_developers, fz214_pct_of_total,
|
||||
escrow_permits, escrow_area_th_sqm, escrow_developers,
|
||||
pp480_permits, pp480_area_th_sqm, pp480_developers)
|
||||
VALUES {','.join(parts[i:i+BATCH])}
|
||||
ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
|
||||
total_permits = EXCLUDED.total_permits,
|
||||
total_area_th_sqm = EXCLUDED.total_area_th_sqm,
|
||||
total_developers = EXCLUDED.total_developers,
|
||||
fz214_permits = EXCLUDED.fz214_permits,
|
||||
fz214_area_th_sqm = EXCLUDED.fz214_area_th_sqm,
|
||||
fz214_developers = EXCLUDED.fz214_developers,
|
||||
fz214_pct_of_total = EXCLUDED.fz214_pct_of_total,
|
||||
escrow_permits = EXCLUDED.escrow_permits,
|
||||
escrow_area_th_sqm = EXCLUDED.escrow_area_th_sqm,
|
||||
escrow_developers = EXCLUDED.escrow_developers,
|
||||
pp480_permits = EXCLUDED.pp480_permits,
|
||||
pp480_area_th_sqm = EXCLUDED.pp480_area_th_sqm,
|
||||
pp480_developers = EXCLUDED.pp480_developers;
|
||||
""")
|
||||
print(f'guaranty_regions: {len(parts)}')
|
||||
|
||||
# ─────────── 4. housing_info.pdf → domrf_housing_summary + planned_commissioning ───────────
|
||||
# extracted manually from PDF
|
||||
housing_summary = [
|
||||
# (territory, level, mechanism, devs, permits, decls, houses, area, flats_th)
|
||||
('Российская Федерация', 'rf', 'all', 4336, 7257, 7447, 11510, 119234, 2424),
|
||||
('Российская Федерация', 'rf', 'escrow', 4285, 7170, 7359, 11385, 117670, 2394),
|
||||
('Российская Федерация', 'rf', 'fund', 42, 61, 62, 85, 1104, 22),
|
||||
('Российская Федерация', 'rf', 'no_attraction', 15, 28, 28, 40, 460, 8),
|
||||
]
|
||||
# top-10 regions (escrow / fund / no_attraction / total) area_th_sqm
|
||||
region_breakdown = [
|
||||
# (region, escrow, fund, no_attr, total)
|
||||
('Город Москва', 15670, 364, 286, 16320),
|
||||
('Краснодарский край', 8179, 24, 4, 8206),
|
||||
('Московская область', 7905, 81, None, 7986),
|
||||
('Свердловская область', 5637, 17, None, 5654),
|
||||
('Город Санкт-Петербург', 5258, None, None, 5258),
|
||||
('Ростовская область', 4926, None, None, 4926),
|
||||
('Ленинградская область', 4640, 60, None, 4700),
|
||||
('Новосибирская область', 3580, 107, None, 3687),
|
||||
('Республика Башкортостан',3583, None, None, 3583),
|
||||
('Тюменская область', 3513, None, 11, 3524),
|
||||
]
|
||||
city_breakdown = [
|
||||
# (city, escrow, fund, no_attr, total)
|
||||
('Москва', 15670, 364, 286, 16320),
|
||||
('Краснодар', 5791, None, None, 5791),
|
||||
('Екатеринбург', 5294, 17, None, 5310),
|
||||
('Санкт-Петербург', 5258, None, None, 5258),
|
||||
('Ростов-на-Дону', 3605, None, None, 3605),
|
||||
('Тюмень', 3204, None, 11, 3215),
|
||||
('Новосибирск', 2755, 104, None, 2859),
|
||||
('Уфа', 2786, None, None, 2786),
|
||||
('Казань', 2205, None, None, 2205),
|
||||
('Владивосток', 2134, None, None, 2134),
|
||||
]
|
||||
parts = []
|
||||
for (terr, level, mech, devs, permits, decls, houses, area, flats) in housing_summary:
|
||||
parts.append(f"('{SNAP_APR26}', {esc(terr)}, '{level}', '{mech}', {integer(devs)}, "
|
||||
f"{integer(permits)}, {integer(decls)}, {integer(houses)}, {num(area)}, {num(flats)})")
|
||||
for breakdown, level in [(region_breakdown, 'region'), (city_breakdown, 'city')]:
|
||||
for (terr, escrow, fund, no_attr, total) in breakdown:
|
||||
for mech, area in [('escrow', escrow), ('fund', fund),
|
||||
('no_attraction', no_attr), ('all', total)]:
|
||||
if area is None: continue
|
||||
parts.append(f"('{SNAP_APR26}', {esc(terr)}, '{level}', '{mech}', NULL, "
|
||||
f"NULL, NULL, NULL, {num(area)}, NULL)")
|
||||
psql(f"""
|
||||
INSERT INTO domrf_housing_summary (snapshot_date, territory_name, territory_level,
|
||||
mechanism, developers_count, permits_count, declarations_count, houses_count,
|
||||
area_thousand_sqm, flats_thousand)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, territory_name, mechanism) DO UPDATE SET
|
||||
developers_count = EXCLUDED.developers_count,
|
||||
permits_count = EXCLUDED.permits_count,
|
||||
declarations_count = EXCLUDED.declarations_count,
|
||||
houses_count = EXCLUDED.houses_count,
|
||||
area_thousand_sqm = EXCLUDED.area_thousand_sqm,
|
||||
flats_thousand = EXCLUDED.flats_thousand;
|
||||
""")
|
||||
print(f'housing_summary: {len(parts)}')
|
||||
|
||||
# planned_commissioning (year, escrow, fund, no_attr, total)
|
||||
pc = [
|
||||
(2026, 37084, 685, 154, 37923),
|
||||
(2027, 34896, 246, 221, 35363),
|
||||
(2028, 26360, 113, 30, 26503),
|
||||
(2029, 10924, 19, 24, 10967),
|
||||
(2030, 4245, 13, 32, 4290),
|
||||
(2031, 1797, 28, None, 1825),
|
||||
(2032, 1323, None, None, 1323),
|
||||
(2033, 351, None, None, 351),
|
||||
(2034, 234, None, None, 234),
|
||||
(2035, 67, None, None, 67),
|
||||
]
|
||||
parts = [f"('{SNAP_APR26}', {y}, {num(e)}, {num(f)}, {num(n)}, {num(t)})"
|
||||
for (y, e, f, n, t) in pc]
|
||||
psql(f"""
|
||||
INSERT INTO domrf_planned_commissioning (snapshot_date, plan_year, escrow_th_sqm,
|
||||
fund_th_sqm, no_attraction_th_sqm, total_th_sqm)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, plan_year) DO UPDATE SET
|
||||
escrow_th_sqm = EXCLUDED.escrow_th_sqm,
|
||||
fund_th_sqm = EXCLUDED.fund_th_sqm,
|
||||
no_attraction_th_sqm = EXCLUDED.no_attraction_th_sqm,
|
||||
total_th_sqm = EXCLUDED.total_th_sqm;
|
||||
""")
|
||||
print(f'planned_commissioning: {len(parts)}')
|
||||
|
||||
# ─────────── 5. sold_out_info.pdf → 3 tables ───────────
|
||||
# RF total
|
||||
psql(f"""
|
||||
INSERT INTO domrf_sold_out (snapshot_date, territory_name, total_area_th_sqm,
|
||||
sold_area_th_sqm, unsold_area_th_sqm, not_open_area_th_sqm,
|
||||
sold_pct, unsold_pct, not_open_pct, avg_price_per_sqm, attracted_funds_mln_rub)
|
||||
VALUES ('{SNAP_SOLD}', 'Российская Федерация',
|
||||
118876, 36749, 53044, 29084, 31, 45, 24, 216316, 7949316)
|
||||
ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
|
||||
total_area_th_sqm = EXCLUDED.total_area_th_sqm,
|
||||
sold_area_th_sqm = EXCLUDED.sold_area_th_sqm,
|
||||
unsold_area_th_sqm = EXCLUDED.unsold_area_th_sqm,
|
||||
not_open_area_th_sqm = EXCLUDED.not_open_area_th_sqm,
|
||||
sold_pct = EXCLUDED.sold_pct,
|
||||
unsold_pct = EXCLUDED.unsold_pct,
|
||||
not_open_pct = EXCLUDED.not_open_pct,
|
||||
avg_price_per_sqm = EXCLUDED.avg_price_per_sqm,
|
||||
attracted_funds_mln_rub = EXCLUDED.attracted_funds_mln_rub;
|
||||
""")
|
||||
# Top-8 regions: (region, total, sold%, unsold%, not_open%)
|
||||
so_regions = [
|
||||
('Город Москва', 16405, 47, 44, None),
|
||||
('Краснодарский край', 8286, 20, 39, 41),
|
||||
('Московская область', 7942, 37, 43, None),
|
||||
('Свердловская область', 5686, 29, 54, None),
|
||||
('Город Санкт-Петербург', 5336, 40, 44, None),
|
||||
('Ростовская область', 4990, None, 48, None),
|
||||
('Ленинградская область', 4705, None, 44, None),
|
||||
('Тюменская область', 3573, None, 50, None),
|
||||
]
|
||||
parts = []
|
||||
for (r, total, sp, up, nop) in so_regions:
|
||||
parts.append(f"('{SNAP_SOLD}', {esc(r)}, {num(total)}, NULL, NULL, NULL, "
|
||||
f"{num(sp)}, {num(up)}, {num(nop)}, NULL, NULL)")
|
||||
psql(f"""
|
||||
INSERT INTO domrf_sold_out (snapshot_date, territory_name, total_area_th_sqm,
|
||||
sold_area_th_sqm, unsold_area_th_sqm, not_open_area_th_sqm,
|
||||
sold_pct, unsold_pct, not_open_pct, avg_price_per_sqm, attracted_funds_mln_rub)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, territory_name) DO UPDATE SET
|
||||
total_area_th_sqm = EXCLUDED.total_area_th_sqm,
|
||||
sold_pct = EXCLUDED.sold_pct,
|
||||
unsold_pct = EXCLUDED.unsold_pct,
|
||||
not_open_pct = EXCLUDED.not_open_pct;
|
||||
""")
|
||||
print(f'sold_out: 1 RF + {len(parts)} regions')
|
||||
|
||||
# by_year — (year, total_th_sqm, pct, sold%, unsold%, not_open%)
|
||||
so_year = [
|
||||
('2026', 40201, 34, 52, 38, 10),
|
||||
('2027', 34880, 29, 30, 53, 17),
|
||||
('2028', 25363, 21, 17, 52, 31),
|
||||
('2029', 10359, 9, None, 37, 54),
|
||||
('2030', 4127, 3, None, None, 71),
|
||||
('2031+', 3946, 3, None, None, 65),
|
||||
]
|
||||
parts = [f"('{SNAP_SOLD}', '{y}', {num(t)}, {num(p)}, {num(s)}, {num(u)}, {num(n)})"
|
||||
for (y, t, p, s, u, n) in so_year]
|
||||
psql(f"""
|
||||
INSERT INTO domrf_sold_out_by_year (snapshot_date, plan_year, total_th_sqm,
|
||||
pct_of_total, sold_pct, unsold_pct, not_open_pct)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, plan_year) DO UPDATE SET
|
||||
total_th_sqm = EXCLUDED.total_th_sqm,
|
||||
pct_of_total = EXCLUDED.pct_of_total,
|
||||
sold_pct = EXCLUDED.sold_pct,
|
||||
unsold_pct = EXCLUDED.unsold_pct,
|
||||
not_open_pct = EXCLUDED.not_open_pct;
|
||||
""")
|
||||
print(f'sold_out_by_year: {len(parts)}')
|
||||
|
||||
# by_progress
|
||||
so_prog = [
|
||||
('not_open', 29084, 24, None, 100),
|
||||
('0-20', 26170, 22, 9, 91),
|
||||
('21-40', 21790, 18, 30, 70),
|
||||
('41-60', 17409, 15, 50, 50),
|
||||
('61-80', 13402, 11, 69, 31),
|
||||
('80+', 11021, 9, 90, 9),
|
||||
]
|
||||
parts = [f"('{SNAP_SOLD}', '{b}', {num(t)}, {num(p)}, {num(s)}, {num(u)})"
|
||||
for (b, t, p, s, u) in so_prog]
|
||||
psql(f"""
|
||||
INSERT INTO domrf_sold_out_by_progress (snapshot_date, progress_bucket, total_th_sqm,
|
||||
pct_of_total, sold_pct_within, unsold_pct_within)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, progress_bucket) DO UPDATE SET
|
||||
total_th_sqm = EXCLUDED.total_th_sqm,
|
||||
pct_of_total = EXCLUDED.pct_of_total,
|
||||
sold_pct_within = EXCLUDED.sold_pct_within,
|
||||
unsold_pct_within = EXCLUDED.unsold_pct_within;
|
||||
""")
|
||||
print(f'sold_out_by_progress: {len(parts)}')
|
||||
|
||||
# ─────────── 6. mortgage_rates.pdf → domrf_mortgage_rates ───────────
|
||||
rates = [
|
||||
# (bank, primary, secondary, refi, note)
|
||||
('Сбербанк', 20.2, 19.7, 21.5, None),
|
||||
('ВТБ', 19.9, 19.9, 19.6, None),
|
||||
('Банк ДОМ.РФ', 18.2, 18.2, 22.1, None),
|
||||
('Альфа-банк', 19.49, 19.49, 18.99, None),
|
||||
('Совкомбанк', 19.99, 20.49, 19.99, None),
|
||||
('Т-Банк', 16.9, 16.9, 16.9, None),
|
||||
('Банк Санкт-Петербург', 18.49, 18.49, 18.49, None),
|
||||
('Промсвязьбанк', 19.49, 18.79, 18.99, None),
|
||||
('Московский Кредитный Банк', 18.6, 18.6, 18.6, None),
|
||||
('Уралсиб', 17.89, 17.89, 18.99, None),
|
||||
('Абсолют банк', 19.35, 19.35, 19.35, None),
|
||||
('Металлинвестбанк', 18.1, 18.1, 18.5, 'к пред.неделе -0.7 пп'),
|
||||
('Газпромбанк', None, None, None, 'Максимальный срок – 19 лет, рефинанс не предоставляет'),
|
||||
('Азиатско-Тихоокеанский Банк', 18.9, 18.9, 18.9, None),
|
||||
('Россельхозбанк', None, None, None, 'не предоставляет'),
|
||||
('Банк Кубань Кредит', 16.4, 16.4, 15.9, None),
|
||||
('Транскапиталбанк', 19.6, 19.6, 19.6, None),
|
||||
('УБРиР', None, None, None, 'не предоставляет'),
|
||||
('Ак Барс Банк', None, None, None, 'ПВ от 50%, рефинанс не предоставляет'),
|
||||
('Средневзвешенная', 19.90, 19.59, 18.81, 'aggregate top-20'),
|
||||
]
|
||||
parts = [f"('{SNAP_MORTGAGE}', {esc(b)}, {num(p)}, {num(s)}, {num(r)}, {esc(n)})"
|
||||
for (b, p, s, r, n) in rates]
|
||||
psql(f"""
|
||||
INSERT INTO domrf_mortgage_rates (snapshot_date, bank_name, primary_rate, secondary_rate, refinance_rate, note)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, bank_name) DO UPDATE SET
|
||||
primary_rate = EXCLUDED.primary_rate,
|
||||
secondary_rate = EXCLUDED.secondary_rate,
|
||||
refinance_rate = EXCLUDED.refinance_rate,
|
||||
note = EXCLUDED.note;
|
||||
""")
|
||||
print(f'mortgage_rates: {len(parts)}')
|
||||
|
||||
# Summary
|
||||
print('\n--- summary ---')
|
||||
out = psql("""
|
||||
SELECT 'developers_full' tab, COUNT(*) FROM domrf_developers_full
|
||||
UNION ALL SELECT 'escrow_banks', COUNT(*) FROM domrf_escrow_banks
|
||||
UNION ALL SELECT 'guaranty_regions', COUNT(*) FROM domrf_guaranty_regions
|
||||
UNION ALL SELECT 'housing_summary', COUNT(*) FROM domrf_housing_summary
|
||||
UNION ALL SELECT 'planned_commissioning', COUNT(*) FROM domrf_planned_commissioning
|
||||
UNION ALL SELECT 'sold_out', COUNT(*) FROM domrf_sold_out
|
||||
UNION ALL SELECT 'sold_out_by_year', COUNT(*) FROM domrf_sold_out_by_year
|
||||
UNION ALL SELECT 'sold_out_by_progress', COUNT(*) FROM domrf_sold_out_by_progress
|
||||
UNION ALL SELECT 'mortgage_rates', COUNT(*) FROM domrf_mortgage_rates
|
||||
ORDER BY 1;
|
||||
""")
|
||||
print(out)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
181
data/sql/30_scrape_domrf.py
Normal file
181
data/sql/30_scrape_domrf.py
Normal file
|
|
@ -0,0 +1,181 @@
|
|||
"""DOM.RF analytics scraper using Playwright (headless Chromium).
|
||||
|
||||
Bypasses:
|
||||
- ServicePipe WAF (Chromium executes JS challenge automatically)
|
||||
- Lazy-loading (wait for `networkidle` + page-specific triggers)
|
||||
- Cookie session (Chromium maintains it across navigations)
|
||||
|
||||
Captures: every JSON response from /api/* endpoints on the same domain.
|
||||
Saves: data/raw/domrf_full/<section>/<endpoint>.json (one JSON per endpoint).
|
||||
|
||||
Usage:
|
||||
PYTHONIOENCODING=utf-8 python data/sql/30_scrape_domrf.py [--headed] [--page <name>]
|
||||
|
||||
Run --headed first to verify WAF passes; switch to default headless after.
|
||||
"""
|
||||
import asyncio, json, os, re, sys
|
||||
from urllib.parse import urlsplit, parse_qs, urlencode, urlunsplit
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(__file__), '..', 'raw', 'domrf_full')
|
||||
BASE = 'https://xn--80az8a.xn--d1aqf.xn--p1ai'
|
||||
|
||||
# 12 analytics pages — name (used as folder) → URL path
|
||||
PAGES = [
|
||||
('housing', '/аналитика/показатели_жилищного_строительства'),
|
||||
('launch', '/аналитика/запуски-и-вводы?repYear=2026&calculationType=SQUARE&period=YEAR'),
|
||||
('share_construction','/аналитика/долевое_строительство'),
|
||||
('housing_dev', '/аналитика/жилищное_строительство'),
|
||||
('escrow', '/аналитика/эскроу'),
|
||||
('realization', '/аналитика/реализация_строящихся_квартир'),
|
||||
('sold_ready', '/аналитика/распроданность-стройготовность?repYear=2026&repMonth=3'),
|
||||
('quartirografia', '/аналитика/квартирография'),
|
||||
('commissioning', '/аналитика/ввод_жилья'),
|
||||
('mortgage_rates', '/аналитика/ставки_предложений_по_ипотеке'),
|
||||
('mortgage_stats', '/аналитика/ипотечное_кредитование'),
|
||||
('stat_series', '/аналитика/статистические_ряды'),
|
||||
]
|
||||
|
||||
|
||||
def safe_filename(url: str) -> str:
|
||||
"""Turn API URL into safe filename: path + query → snake_case."""
|
||||
parts = urlsplit(url)
|
||||
base = parts.path.split('/api/')[-1].strip('/')
|
||||
base = re.sub(r'[^A-Za-z0-9_\-]', '_', base)[:100]
|
||||
if parts.query:
|
||||
q = re.sub(r'[^A-Za-z0-9_\-]', '_', parts.query)[:80]
|
||||
base = f'{base}__{q}'
|
||||
return base + '.json'
|
||||
|
||||
|
||||
def upgrade_size(url: str) -> str:
|
||||
"""If URL has size=15 (or any small value), upgrade to size=1000 to get full data."""
|
||||
parts = urlsplit(url)
|
||||
if not parts.query:
|
||||
return url
|
||||
qs = parse_qs(parts.query, keep_blank_values=True)
|
||||
if 'size' in qs:
|
||||
qs['size'] = ['1000']
|
||||
new_q = urlencode({k: v[0] for k, v in qs.items()})
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, new_q, parts.fragment))
|
||||
|
||||
|
||||
async def scrape_page(browser, name, path):
|
||||
out_dir = os.path.join(ROOT, name)
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
print(f'\n=== {name} ({path}) ===')
|
||||
context = await browser.new_context(
|
||||
user_agent='Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 '
|
||||
'(KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36',
|
||||
locale='ru-RU',
|
||||
viewport={'width': 1920, 'height': 1080},
|
||||
)
|
||||
page = await context.new_page()
|
||||
|
||||
seen_urls = set()
|
||||
saved = []
|
||||
|
||||
async def on_response(resp):
|
||||
url = resp.url
|
||||
# Match same-origin /api/ endpoints, skip yandex/sendsay trackers
|
||||
if 'xn--80az8a.xn--d1aqf.xn--p1ai' not in url or '/api/' not in url:
|
||||
return
|
||||
ct = (resp.headers.get('content-type') or '').lower()
|
||||
if 'json' not in ct:
|
||||
return
|
||||
if url in seen_urls:
|
||||
return
|
||||
seen_urls.add(url)
|
||||
try:
|
||||
text = await resp.text()
|
||||
fname = safe_filename(url)
|
||||
with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
|
||||
f.write(text)
|
||||
saved.append((fname, len(text), url))
|
||||
print(f' saved {fname} ({len(text)}b)')
|
||||
except Exception as e:
|
||||
print(f' ERR {url}: {e}')
|
||||
|
||||
page.on('response', lambda r: asyncio.create_task(on_response(r)))
|
||||
|
||||
full_url = BASE + path
|
||||
try:
|
||||
await page.goto(full_url, wait_until='domcontentloaded', timeout=60_000)
|
||||
# WAF challenge usually completes within 1-3 seconds
|
||||
try:
|
||||
await page.wait_for_load_state('networkidle', timeout=30_000)
|
||||
except:
|
||||
pass
|
||||
# Extra wait for lazy-loaded API calls
|
||||
await page.wait_for_timeout(8000)
|
||||
# Scroll multiple times to trigger any IntersectionObserver-bound fetches
|
||||
for _ in range(3):
|
||||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight)')
|
||||
await page.wait_for_timeout(1500)
|
||||
await page.evaluate('window.scrollTo(0, document.body.scrollHeight / 2)')
|
||||
await page.wait_for_timeout(1500)
|
||||
await page.evaluate('window.scrollTo(0, 0)')
|
||||
await page.wait_for_timeout(1500)
|
||||
# Click first interactive button (triggers React state → API)
|
||||
try:
|
||||
await page.evaluate('''() => {
|
||||
const btn = document.querySelector('button:not([disabled])');
|
||||
if (btn) btn.click();
|
||||
}''')
|
||||
await page.wait_for_timeout(3000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Re-fetch with size=1000 for endpoints that have small page-size
|
||||
for url in list(seen_urls):
|
||||
up = upgrade_size(url)
|
||||
if up != url and up not in seen_urls:
|
||||
try:
|
||||
resp = await page.evaluate(
|
||||
'''(u) => fetch(u, {credentials:'include'})
|
||||
.then(r => r.text()).then(t => ({status: 200, text: t}))''',
|
||||
up,
|
||||
)
|
||||
if resp and resp.get('text'):
|
||||
seen_urls.add(up)
|
||||
fname = safe_filename(up)
|
||||
with open(os.path.join(out_dir, fname), 'w', encoding='utf-8') as f:
|
||||
f.write(resp['text'])
|
||||
saved.append((fname, len(resp['text']), up))
|
||||
print(f' upgraded {fname} ({len(resp["text"])}b)')
|
||||
except Exception as e:
|
||||
print(f' upgrade err {up}: {e}')
|
||||
finally:
|
||||
await context.close()
|
||||
return saved
|
||||
|
||||
|
||||
async def main():
|
||||
headed = '--headed' in sys.argv
|
||||
target = None
|
||||
if '--page' in sys.argv:
|
||||
target = sys.argv[sys.argv.index('--page') + 1]
|
||||
|
||||
pages = [(n, p) for n, p in PAGES if target is None or n == target]
|
||||
print(f'Scraping {len(pages)} pages, headed={headed}')
|
||||
os.makedirs(ROOT, exist_ok=True)
|
||||
|
||||
async with async_playwright() as pw:
|
||||
browser = await pw.chromium.launch(headless=not headed)
|
||||
all_saved = {}
|
||||
for name, path in pages:
|
||||
try:
|
||||
all_saved[name] = await scrape_page(browser, name, path)
|
||||
except Exception as e:
|
||||
print(f'PAGE {name} FAILED: {e}')
|
||||
all_saved[name] = []
|
||||
await browser.close()
|
||||
|
||||
print('\n=== SUMMARY ===')
|
||||
for name, saved in all_saved.items():
|
||||
total = sum(s[1] for s in saved)
|
||||
print(f' {name:22s} files={len(saved):3d} total={total:>10}b')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
asyncio.run(main())
|
||||
166
data/sql/31_schema_domrf_full.sql
Normal file
166
data/sql/31_schema_domrf_full.sql
Normal file
|
|
@ -0,0 +1,166 @@
|
|||
-- DOM.RF analytics, normalized schema for scraper output (data/raw/domrf_full/*).
|
||||
-- Snapshot date stamped per scrape; idempotent UPSERT loaders.
|
||||
|
||||
-- ── 1. Launch (запуски + ввод в эксплуатацию) ────────────────────────────────
|
||||
-- Topology: page returns top-N entities (developers/regions/areas/fo) with launch (app)
|
||||
-- and commissioning (rnv) values. Single table covers all 4 dimensions via dim_type.
|
||||
CREATE TABLE IF NOT EXISTS domrf_launch_top (
|
||||
snapshot_date DATE NOT NULL,
|
||||
rep_year INT NOT NULL,
|
||||
calc_type TEXT NOT NULL, -- 'SQUARE' (м²) | 'HOUSES' (шт) | 'FLATS' (шт)
|
||||
dim_type TEXT NOT NULL, -- 'developer' | 'region' | 'area' | 'fo'
|
||||
metric_type TEXT NOT NULL, -- 'app' (запуски) | 'rnv' (ввод)
|
||||
entity_id TEXT NOT NULL, -- '6072_0' (developer) | '50' (region) | 'Москва' (area)
|
||||
entity_name TEXT NOT NULL,
|
||||
value NUMERIC NOT NULL,
|
||||
PRIMARY KEY (snapshot_date, rep_year, calc_type, dim_type, metric_type, entity_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_launch_top_dim ON domrf_launch_top (dim_type, metric_type, value DESC);
|
||||
|
||||
-- Объёмы по классам жилья
|
||||
CREATE TABLE IF NOT EXISTS domrf_launch_obj_class (
|
||||
snapshot_date DATE NOT NULL,
|
||||
rep_year INT NOT NULL,
|
||||
calc_type TEXT NOT NULL,
|
||||
obj_class_cd INT NOT NULL, -- 1=Типовой, 2=Комфорт, 3=Бизнес, 4=Элитный
|
||||
obj_class_desc TEXT NOT NULL,
|
||||
app_value NUMERIC,
|
||||
rnv_value NUMERIC,
|
||||
PRIMARY KEY (snapshot_date, rep_year, calc_type, obj_class_cd)
|
||||
);
|
||||
|
||||
-- Time series: год × месяц → запуски/вводы
|
||||
CREATE TABLE IF NOT EXISTS domrf_launch_monthly (
|
||||
snapshot_date DATE NOT NULL,
|
||||
calc_type TEXT NOT NULL,
|
||||
rep_year INT NOT NULL,
|
||||
rep_month INT NOT NULL, -- 1..12
|
||||
app_value NUMERIC,
|
||||
rnv_value NUMERIC,
|
||||
PRIMARY KEY (snapshot_date, calc_type, rep_year, rep_month)
|
||||
);
|
||||
|
||||
-- ── 2. Реализация / распроданность-стройготовность (sold_ready) ──────────────
|
||||
CREATE TABLE IF NOT EXISTS domrf_sold_ready_index (
|
||||
snapshot_date DATE NOT NULL,
|
||||
rep_year INT NOT NULL,
|
||||
rep_month INT NOT NULL,
|
||||
square_sum NUMERIC, -- всего стр. площади РФ, м²
|
||||
sold_perc NUMERIC, -- % проданного
|
||||
sold_sum NUMERIC, -- проданная площадь, м²
|
||||
sold_ready_perc NUMERIC, -- "распроданность ÷ готовность"
|
||||
ready_perc NUMERIC, -- % готовности
|
||||
PRIMARY KEY (snapshot_date, rep_year, rep_month)
|
||||
);
|
||||
|
||||
-- Reality breakdowns: foChart / regionChart / cityChart / devChart …
|
||||
CREATE TABLE IF NOT EXISTS domrf_sold_ready_breakdown (
|
||||
snapshot_date DATE NOT NULL,
|
||||
rep_year INT NOT NULL,
|
||||
rep_month INT NOT NULL,
|
||||
chart_type TEXT NOT NULL, -- 'foChart' | 'regionChart' | 'cityChart' | 'devChart' | ...
|
||||
entity_key TEXT NOT NULL, -- название/код
|
||||
square_sum NUMERIC,
|
||||
sold_perc NUMERIC,
|
||||
ready_perc NUMERIC,
|
||||
sold_ready_perc NUMERIC,
|
||||
fo_cd INT,
|
||||
extra JSONB, -- backup для прочих полей
|
||||
PRIMARY KEY (snapshot_date, rep_year, rep_month, chart_type, entity_key)
|
||||
);
|
||||
|
||||
-- Динамика: chartType × год × месяц → значение
|
||||
CREATE TABLE IF NOT EXISTS domrf_sold_ready_dynamics (
|
||||
snapshot_date DATE NOT NULL,
|
||||
dynamic_chart_type TEXT NOT NULL, -- 'squareSumChart' | 'soldPercChart' | 'readyPercChart' | 'soldReadyPercChart'
|
||||
rep_year INT NOT NULL,
|
||||
rep_month INT NOT NULL,
|
||||
value NUMERIC,
|
||||
PRIMARY KEY (snapshot_date, dynamic_chart_type, rep_year, rep_month)
|
||||
);
|
||||
|
||||
-- ── 3. Share construction (долевое стр-во / project finance) ────────────────
|
||||
-- /аналитика/api/project/finance/dashboard returns rows per region/FO/RF.
|
||||
CREATE TABLE IF NOT EXISTS domrf_project_finance (
|
||||
snapshot_date DATE NOT NULL,
|
||||
report_date DATE,
|
||||
subject_type TEXT, -- 'rf' | 'fo' | 'region'
|
||||
subject_name TEXT,
|
||||
region_cd INT,
|
||||
fo_cd INT,
|
||||
fo_desc TEXT,
|
||||
rns_cnt INT, -- разрешения на стр-во
|
||||
dev_cnt INT, -- застройщики
|
||||
liv_sq_amt NUMERIC, -- жилая площадь м²
|
||||
guaranty_escrow_rns_cnt INT, -- эскроу разрешения
|
||||
guaranty_escrow_dev_cnt INT,
|
||||
guaranty_escrow_liv_sq_amt NUMERIC,
|
||||
guaranty_zosg_rns_cnt INT,
|
||||
guaranty_zosg_dev_cnt INT,
|
||||
guaranty_zosg_liv_sq_amt NUMERIC,
|
||||
guaranty_rns_cnt INT, -- всего по 214-ФЗ
|
||||
guaranty_dev_cnt INT,
|
||||
guaranty_liv_sq_amt NUMERIC,
|
||||
nonguaranty_rns_cnt INT, -- ПП №480
|
||||
nonguaranty_dev_cnt INT,
|
||||
nonguaranty_liv_sq_amt NUMERIC,
|
||||
extra JSONB,
|
||||
id BIGSERIAL PRIMARY KEY
|
||||
);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS uq_project_finance_snap
|
||||
ON domrf_project_finance (snapshot_date, COALESCE(subject_type, ''), COALESCE(subject_name, ''));
|
||||
|
||||
-- ── 4. Ввод жилья (commissioning building_summary) ──────────────────────────
|
||||
CREATE TABLE IF NOT EXISTS domrf_commissioning (
|
||||
snapshot_date DATE NOT NULL,
|
||||
region_id INT NOT NULL,
|
||||
region_name TEXT NOT NULL,
|
||||
region_type TEXT, -- 'rf' | 'fo' | 'region'
|
||||
reporting_period DATE NOT NULL,
|
||||
rep_year INT,
|
||||
accumulated_fact_area NUMERIC,
|
||||
accumulated_fact_area_multifamily NUMERIC,
|
||||
accumulated_fact_area_private NUMERIC,
|
||||
accumulated_fact_area_change NUMERIC,
|
||||
accumulated_fact_area_change_share NUMERIC,
|
||||
extra JSONB,
|
||||
PRIMARY KEY (snapshot_date, region_id, reporting_period)
|
||||
);
|
||||
|
||||
-- ── 5. Mortgage stats ───────────────────────────────────────────────────────
|
||||
CREATE TABLE IF NOT EXISTS domrf_mortgage_dashboard (
|
||||
snapshot_date DATE NOT NULL PRIMARY KEY,
|
||||
total_credit_count INT,
|
||||
total_credit_count_delta_pct NUMERIC,
|
||||
primary_credit_count INT,
|
||||
primary_credit_count_delta_pct NUMERIC,
|
||||
secondary_credit_count INT,
|
||||
secondary_credit_count_delta_pct NUMERIC,
|
||||
total_credit_amount NUMERIC,
|
||||
total_credit_amount_delta_pct NUMERIC,
|
||||
primary_credit_amount NUMERIC,
|
||||
primary_credit_amount_delta_pct NUMERIC,
|
||||
secondary_credit_amount NUMERIC,
|
||||
secondary_credit_amount_delta_pct NUMERIC,
|
||||
total_credit_avg_rate NUMERIC,
|
||||
total_credit_avg_rate_delta NUMERIC,
|
||||
primary_credit_avg_rate NUMERIC,
|
||||
primary_credit_avg_rate_delta NUMERIC,
|
||||
secondary_credit_avg_rate NUMERIC,
|
||||
secondary_credit_avg_rate_delta NUMERIC,
|
||||
extra JSONB
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS domrf_mortgage_details (
|
||||
snapshot_date DATE NOT NULL,
|
||||
currency TEXT NOT NULL,
|
||||
credit_amount_avg NUMERIC,
|
||||
credit_amount_avg_delta_pct NUMERIC,
|
||||
credit_avg_period NUMERIC,
|
||||
credit_avg_period_delta_pct NUMERIC,
|
||||
credit_debts_amount NUMERIC,
|
||||
credit_debts_amount_delta_pct NUMERIC,
|
||||
credit_debts_overdue_percent NUMERIC,
|
||||
credit_debts_overdue_percent_delta NUMERIC,
|
||||
PRIMARY KEY (snapshot_date, currency)
|
||||
);
|
||||
18
data/sql/31_schema_domrf_raw.sql
Normal file
18
data/sql/31_schema_domrf_raw.sql
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
-- DOM.RF analytics raw API endpoint snapshots.
|
||||
-- Universal JSONB store: one row per (snapshot_date, section, endpoint).
|
||||
-- Section = page name on наш.дом.рф (launch / share_construction / escrow / ...).
|
||||
-- Endpoint = sanitized path+query of /api/* call.
|
||||
|
||||
CREATE TABLE IF NOT EXISTS domrf_raw_endpoints (
|
||||
snapshot_date DATE NOT NULL,
|
||||
section TEXT NOT NULL, -- 'launch' | 'sold_ready' | 'quartirografia' | ...
|
||||
endpoint TEXT NOT NULL, -- 'launch_developers_app__calculationType_SQUARE_repYear_2026_size_1000'
|
||||
source_url TEXT, -- inferred original URL (best effort)
|
||||
payload JSONB NOT NULL,
|
||||
payload_size INT, -- bytes (raw text length)
|
||||
fetched_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||||
PRIMARY KEY (snapshot_date, section, endpoint)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_domrf_raw_section ON domrf_raw_endpoints (section);
|
||||
CREATE INDEX IF NOT EXISTS idx_domrf_raw_payload_gin ON domrf_raw_endpoints USING GIN (payload);
|
||||
378
data/sql/32_load_domrf_full.py
Normal file
378
data/sql/32_load_domrf_full.py
Normal file
|
|
@ -0,0 +1,378 @@
|
|||
"""Loads scraped DOM.RF JSONs into normalized PG tables.
|
||||
|
||||
Reads: data/raw/domrf_full/<section>/*.json
|
||||
Writes: domrf_launch_top, ..._obj_class, ..._monthly,
|
||||
domrf_sold_ready_index, ..._breakdown, ..._dynamics,
|
||||
domrf_project_finance, domrf_commissioning,
|
||||
domrf_mortgage_dashboard, domrf_mortgage_details.
|
||||
|
||||
Existing tables (domrf_developers, domrf_regions, domrf_developer_aggregates,
|
||||
domrf_region_aggregates, domrf_developers_full, domrf_escrow_banks,
|
||||
domrf_guaranty_regions, domrf_housing_summary, domrf_sold_out,
|
||||
domrf_mortgage_rates, etc.) are NOT touched.
|
||||
"""
|
||||
import os, json, sys, subprocess, datetime
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.join(HERE, '..', 'raw', 'domrf_full')
|
||||
PG_PORT = os.environ.get('PG_PORT', '5432')
|
||||
PG_USER = os.environ.get('PG_USER', 'gendesign')
|
||||
PG_DB = os.environ.get('PG_DB', 'gendesign')
|
||||
PG_PASS = os.environ.get('PGPASSWORD', '')
|
||||
SNAP = os.environ.get('SNAPSHOT_DATE', datetime.date.today().isoformat())
|
||||
|
||||
|
||||
def psql(sql):
|
||||
cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
|
||||
'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT,
|
||||
'-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet']
|
||||
res = subprocess.run(cmd, input=sql, capture_output=True, text=True, encoding='utf-8')
|
||||
if res.returncode != 0:
|
||||
print('psql FAILED:', res.stderr[-2000:], file=sys.stderr)
|
||||
raise SystemExit(res.returncode)
|
||||
return res.stdout
|
||||
|
||||
|
||||
def esc(s):
|
||||
if s is None: return 'NULL'
|
||||
return "'" + str(s).replace("'", "''") + "'"
|
||||
|
||||
def num(v):
|
||||
if v is None or v == '-' or v == '': return 'NULL'
|
||||
try: return str(float(v))
|
||||
except (ValueError, TypeError): return 'NULL'
|
||||
|
||||
def integer(v):
|
||||
if v is None: return 'NULL'
|
||||
try: return str(int(float(v)))
|
||||
except (ValueError, TypeError): return 'NULL'
|
||||
|
||||
|
||||
def load_json(section, fname):
|
||||
p = os.path.join(ROOT, section, fname + '.json')
|
||||
if not os.path.exists(p): return None
|
||||
return json.load(open(p, encoding='utf-8'))
|
||||
|
||||
|
||||
def find_files(section, prefix):
|
||||
"""Find files in section/ starting with prefix (e.g. 'launch_developers_app__')."""
|
||||
sec_dir = os.path.join(ROOT, section)
|
||||
if not os.path.isdir(sec_dir): return []
|
||||
return [f for f in os.listdir(sec_dir) if f.startswith(prefix) and f.endswith('.json')]
|
||||
|
||||
|
||||
def best_size_file(section, prefix):
|
||||
"""Pick the file with largest size= parameter (or only one)."""
|
||||
files = find_files(section, prefix)
|
||||
if not files: return None
|
||||
# Prefer size=1000 over size=15
|
||||
files.sort(key=lambda f: 0 if 'size_1000' in f else 1)
|
||||
return files[0][:-5] # strip .json
|
||||
|
||||
|
||||
def main():
|
||||
schema = open(os.path.join(HERE, '31_schema_domrf_full.sql'), encoding='utf-8').read()
|
||||
psql(schema)
|
||||
print('schema applied')
|
||||
|
||||
# ── 1. LAUNCH ────────────────────────────────────────────────────────────
|
||||
rep_year = 2026
|
||||
parts = []
|
||||
for dim, prefix_app, prefix_rnv in [
|
||||
('developer', 'launch_developers_app__', 'launch_developers_rnv__'),
|
||||
('region', 'launch_regions_app__', 'launch_regions_rnv__'),
|
||||
('area', 'launch_areas_app__', 'launch_areas_rnv__'),
|
||||
('fo', 'launch_federal-districts_app__', 'launch_federal-districts_rnv__'),
|
||||
]:
|
||||
for metric, prefix in [('app', prefix_app), ('rnv', prefix_rnv)]:
|
||||
fname = best_size_file('launch', prefix)
|
||||
if not fname: continue
|
||||
d = load_json('launch', fname)
|
||||
calc = d.get('calculationType', 'SQUARE')
|
||||
for v in d.get('values', []):
|
||||
eid = str(v.get('id', v.get('name', '')))
|
||||
ename = v.get('name', eid)
|
||||
value = v.get('value')
|
||||
if value is None: continue
|
||||
parts.append(f"('{SNAP}', {rep_year}, {esc(calc)}, '{dim}', '{metric}', "
|
||||
f"{esc(eid)}, {esc(ename)}, {num(value)})")
|
||||
if parts:
|
||||
for i in range(0, len(parts), 500):
|
||||
psql(f"""
|
||||
INSERT INTO domrf_launch_top (snapshot_date, rep_year, calc_type, dim_type, metric_type, entity_id, entity_name, value)
|
||||
VALUES {','.join(parts[i:i+500])}
|
||||
ON CONFLICT (snapshot_date, rep_year, calc_type, dim_type, metric_type, entity_id) DO UPDATE SET
|
||||
entity_name = EXCLUDED.entity_name, value = EXCLUDED.value;
|
||||
""")
|
||||
print(f'launch_top: {len(parts)}')
|
||||
|
||||
# obj_class
|
||||
d = load_json('launch', 'launch_obj-class_table__calculationType_SQUARE_repYear_2026') \
|
||||
or load_json('launch', 'launch_obj-class_table')
|
||||
if d:
|
||||
calc = d.get('calculationType', 'SQUARE')
|
||||
parts = [f"('{SNAP}', {rep_year}, {esc(calc)}, {integer(o['objClassCd'])}, "
|
||||
f"{esc(o['objClassDesc'])}, {num(o.get('appValue'))}, {num(o.get('rnvValue'))})"
|
||||
for o in d.get('objClassTypes', [])]
|
||||
if parts:
|
||||
psql(f"""
|
||||
INSERT INTO domrf_launch_obj_class (snapshot_date, rep_year, calc_type, obj_class_cd, obj_class_desc, app_value, rnv_value)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, rep_year, calc_type, obj_class_cd) DO UPDATE SET
|
||||
obj_class_desc = EXCLUDED.obj_class_desc,
|
||||
app_value = EXCLUDED.app_value, rnv_value = EXCLUDED.rnv_value;
|
||||
""")
|
||||
print(f'launch_obj_class: {len(parts)}')
|
||||
|
||||
# monthly time series
|
||||
d = load_json('launch', 'charts')
|
||||
if d:
|
||||
calc = d.get('calculationType', 'SQUARE')
|
||||
parts = []
|
||||
for y in d.get('repYears', []):
|
||||
yr = y['repYear']
|
||||
for m in y.get('repMonth', []):
|
||||
parts.append(f"('{SNAP}', {esc(calc)}, {yr}, {integer(m['repMonth'])}, "
|
||||
f"{num(m.get('appValue'))}, {num(m.get('rnvValue'))})")
|
||||
if parts:
|
||||
for i in range(0, len(parts), 500):
|
||||
psql(f"""
|
||||
INSERT INTO domrf_launch_monthly (snapshot_date, calc_type, rep_year, rep_month, app_value, rnv_value)
|
||||
VALUES {','.join(parts[i:i+500])}
|
||||
ON CONFLICT (snapshot_date, calc_type, rep_year, rep_month) DO UPDATE SET
|
||||
app_value = EXCLUDED.app_value, rnv_value = EXCLUDED.rnv_value;
|
||||
""")
|
||||
print(f'launch_monthly: {len(parts)}')
|
||||
|
||||
# ── 2. SOLD READY ────────────────────────────────────────────────────────
|
||||
rep_year, rep_month = 2026, 3
|
||||
d = load_json('sold_ready', 'ready-construction_index__repMonth_3_repYear_2026')
|
||||
if d:
|
||||
psql(f"""
|
||||
INSERT INTO domrf_sold_ready_index (snapshot_date, rep_year, rep_month, square_sum, sold_perc, sold_sum, sold_ready_perc, ready_perc)
|
||||
VALUES ('{SNAP}', {rep_year}, {rep_month}, {num(d.get('squareSumIndex'))},
|
||||
{num(d.get('soldPercIndex'))}, {num(d.get('soldSumIndex'))},
|
||||
{num(d.get('soldReadyPercIndex'))}, {num(d.get('readyPercIndex'))})
|
||||
ON CONFLICT (snapshot_date, rep_year, rep_month) DO UPDATE SET
|
||||
square_sum = EXCLUDED.square_sum, sold_perc = EXCLUDED.sold_perc,
|
||||
sold_sum = EXCLUDED.sold_sum, sold_ready_perc = EXCLUDED.sold_ready_perc,
|
||||
ready_perc = EXCLUDED.ready_perc;
|
||||
""")
|
||||
print('sold_ready_index: 1')
|
||||
|
||||
d = load_json('sold_ready', 'ready-construction_charts__repMonth_3_repYear_2026')
|
||||
if d:
|
||||
parts = []
|
||||
seen = set()
|
||||
for chart in d.get('charts', []):
|
||||
ct = chart.get('chartType')
|
||||
for row in chart.get('data', []):
|
||||
key = row.get('key', '')
|
||||
if not key: continue
|
||||
pk = (ct, key)
|
||||
if pk in seen: continue
|
||||
seen.add(pk)
|
||||
extra = {k: v for k, v in row.items() if k not in
|
||||
{'key', 'squareSum', 'soldPerc', 'readyPerc', 'soldReadyPerc', 'foCd'}}
|
||||
extra_json = json.dumps(extra, ensure_ascii=False) if extra else None
|
||||
parts.append(f"('{SNAP}', {rep_year}, {rep_month}, {esc(ct)}, {esc(key)}, "
|
||||
f"{num(row.get('squareSum'))}, {num(row.get('soldPerc'))}, "
|
||||
f"{num(row.get('readyPerc'))}, {num(row.get('soldReadyPerc'))}, "
|
||||
f"{integer(row.get('foCd'))}, "
|
||||
f"{('NULL' if extra_json is None else esc(extra_json) + '::jsonb')})")
|
||||
if parts:
|
||||
for i in range(0, len(parts), 500):
|
||||
psql(f"""
|
||||
INSERT INTO domrf_sold_ready_breakdown (snapshot_date, rep_year, rep_month, chart_type, entity_key, square_sum, sold_perc, ready_perc, sold_ready_perc, fo_cd, extra)
|
||||
VALUES {','.join(parts[i:i+500])}
|
||||
ON CONFLICT (snapshot_date, rep_year, rep_month, chart_type, entity_key) DO UPDATE SET
|
||||
square_sum = EXCLUDED.square_sum, sold_perc = EXCLUDED.sold_perc,
|
||||
ready_perc = EXCLUDED.ready_perc, sold_ready_perc = EXCLUDED.sold_ready_perc,
|
||||
fo_cd = EXCLUDED.fo_cd, extra = EXCLUDED.extra;
|
||||
""")
|
||||
print(f'sold_ready_breakdown: {len(parts)}')
|
||||
|
||||
d = load_json('sold_ready', 'ready-construction_dynamics__dynamicChartType_readyPercChart_2CsoldPercChart_2CsoldReadyPercChart_2CsquareSum')
|
||||
if d:
|
||||
parts = []
|
||||
for chart in d.get('dynamicCharts', []):
|
||||
ct = chart.get('dynamicChartType')
|
||||
for y in chart.get('repYears', []):
|
||||
yr = y.get('repYear')
|
||||
for m in y.get('repMonths', []):
|
||||
parts.append(f"('{SNAP}', {esc(ct)}, {integer(yr)}, {integer(m.get('repMonth'))}, {num(m.get('value'))})")
|
||||
if parts:
|
||||
for i in range(0, len(parts), 500):
|
||||
psql(f"""
|
||||
INSERT INTO domrf_sold_ready_dynamics (snapshot_date, dynamic_chart_type, rep_year, rep_month, value)
|
||||
VALUES {','.join(parts[i:i+500])}
|
||||
ON CONFLICT (snapshot_date, dynamic_chart_type, rep_year, rep_month) DO UPDATE SET
|
||||
value = EXCLUDED.value;
|
||||
""")
|
||||
print(f'sold_ready_dynamics: {len(parts)}')
|
||||
|
||||
# ── 3. PROJECT FINANCE (share_construction) ─────────────────────────────
|
||||
d = load_json('share_construction', 'dashboard') or load_json('share_construction', 'project_finance_dashboard__dateDay_26-04-2026')
|
||||
if d:
|
||||
rows = d.get('data', {}).get('data', []) if isinstance(d.get('data'), dict) else d.get('data', [])
|
||||
parts = []
|
||||
for row in rows:
|
||||
extra = {k: v for k, v in row.items() if k not in {
|
||||
'id','subjectType','subject','regionCd','foCd','foDesc',
|
||||
'rnsCnt','devCnt','livSqAmt',
|
||||
'guarantyEscrowRnsCnt','guarantyEscrowDevCnt','guarantyEscrowLivSqAmt',
|
||||
'guarantyZosgRnsCnt','guarantyZosgDevCnt','guarantyZosgLivSqAmt',
|
||||
'guarantyRnsCnt','guarantyDevCnt','guarantyLivSqAmt',
|
||||
'nonguarantyRnsCnt','nonguarantyDevCnt','nonguarantyLivSqAmt'}}
|
||||
parts.append((
|
||||
row.get('subjectType'), row.get('subject'),
|
||||
row.get('regionCd'), row.get('foCd'), row.get('foDesc'),
|
||||
row.get('rnsCnt'), row.get('devCnt'), row.get('livSqAmt'),
|
||||
row.get('guarantyEscrowRnsCnt'), row.get('guarantyEscrowDevCnt'), row.get('guarantyEscrowLivSqAmt'),
|
||||
row.get('guarantyZosgRnsCnt'), row.get('guarantyZosgDevCnt'), row.get('guarantyZosgLivSqAmt'),
|
||||
row.get('guarantyRnsCnt'), row.get('guarantyDevCnt'), row.get('guarantyLivSqAmt'),
|
||||
row.get('nonguarantyRnsCnt'), row.get('nonguarantyDevCnt'), row.get('nonguarantyLivSqAmt'),
|
||||
json.dumps(extra, ensure_ascii=False) if extra else None,
|
||||
))
|
||||
if parts:
|
||||
# Clear today's data first (since PK is by subject_type+name, may have duplicates by region within fo)
|
||||
psql(f"DELETE FROM domrf_project_finance WHERE snapshot_date = '{SNAP}';")
|
||||
sql_parts = []
|
||||
for p in parts:
|
||||
(st, sn, rc, fc, fd, rn, dn, ls, gers, gerd, gerls,
|
||||
gzrs, gzrd, gzrls, grns, grdv, grls, nrns, nrdv, nrls, ej) = p
|
||||
sql_parts.append(
|
||||
f"('{SNAP}', NULL, {esc(st)}, {esc(sn)}, {integer(rc)}, {integer(fc)}, {esc(fd)}, "
|
||||
f"{integer(rn)}, {integer(dn)}, {num(ls)}, "
|
||||
f"{integer(gers)}, {integer(gerd)}, {num(gerls)}, "
|
||||
f"{integer(gzrs)}, {integer(gzrd)}, {num(gzrls)}, "
|
||||
f"{integer(grns)}, {integer(grdv)}, {num(grls)}, "
|
||||
f"{integer(nrns)}, {integer(nrdv)}, {num(nrls)}, "
|
||||
f"{('NULL' if ej is None else esc(ej) + '::jsonb')})"
|
||||
)
|
||||
for i in range(0, len(sql_parts), 200):
|
||||
psql(f"""
|
||||
INSERT INTO domrf_project_finance (snapshot_date, report_date, subject_type, subject_name,
|
||||
region_cd, fo_cd, fo_desc, rns_cnt, dev_cnt, liv_sq_amt,
|
||||
guaranty_escrow_rns_cnt, guaranty_escrow_dev_cnt, guaranty_escrow_liv_sq_amt,
|
||||
guaranty_zosg_rns_cnt, guaranty_zosg_dev_cnt, guaranty_zosg_liv_sq_amt,
|
||||
guaranty_rns_cnt, guaranty_dev_cnt, guaranty_liv_sq_amt,
|
||||
nonguaranty_rns_cnt, nonguaranty_dev_cnt, nonguaranty_liv_sq_amt, extra)
|
||||
VALUES {','.join(sql_parts[i:i+200])}
|
||||
ON CONFLICT DO NOTHING;
|
||||
""")
|
||||
print(f'project_finance: {len(parts)}')
|
||||
|
||||
# ── 4. COMMISSIONING ───────────────────────────────────────────────────
|
||||
d = load_json('commissioning', 'building_summary_data')
|
||||
if d:
|
||||
rows = d.get('data', [])
|
||||
parts = []
|
||||
seen = set()
|
||||
for r in rows:
|
||||
pk = (r.get('regionId'), r.get('reportingPeriod'))
|
||||
if pk in seen: continue
|
||||
seen.add(pk)
|
||||
extra = {k: v for k, v in r.items() if k not in {
|
||||
'regionId','regionName','regionType','reportingPeriod','year',
|
||||
'accumulatedFactArea','accumulatedFactAreaMultifamily','accumulatedFactAreaPrivate',
|
||||
'accumulatedFactAreaChange','accumulatedFactAreaChangeShare'}}
|
||||
parts.append(f"('{SNAP}', {integer(r.get('regionId'))}, {esc(r.get('regionName'))}, "
|
||||
f"{esc(r.get('regionType'))}, {esc(r.get('reportingPeriod'))}::date, "
|
||||
f"{integer(r.get('year'))}, "
|
||||
f"{num(r.get('accumulatedFactArea'))}, "
|
||||
f"{num(r.get('accumulatedFactAreaMultifamily'))}, "
|
||||
f"{num(r.get('accumulatedFactAreaPrivate'))}, "
|
||||
f"{num(r.get('accumulatedFactAreaChange'))}, "
|
||||
f"{num(r.get('accumulatedFactAreaChangeShare'))}, "
|
||||
f"{esc(json.dumps(extra, ensure_ascii=False)) + '::jsonb' if extra else 'NULL'})")
|
||||
if parts:
|
||||
psql(f"""
|
||||
INSERT INTO domrf_commissioning (snapshot_date, region_id, region_name, region_type,
|
||||
reporting_period, rep_year, accumulated_fact_area, accumulated_fact_area_multifamily,
|
||||
accumulated_fact_area_private, accumulated_fact_area_change, accumulated_fact_area_change_share, extra)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, region_id, reporting_period) DO UPDATE SET
|
||||
region_name = EXCLUDED.region_name,
|
||||
accumulated_fact_area = EXCLUDED.accumulated_fact_area,
|
||||
accumulated_fact_area_multifamily = EXCLUDED.accumulated_fact_area_multifamily,
|
||||
accumulated_fact_area_private = EXCLUDED.accumulated_fact_area_private,
|
||||
accumulated_fact_area_change = EXCLUDED.accumulated_fact_area_change,
|
||||
accumulated_fact_area_change_share = EXCLUDED.accumulated_fact_area_change_share;
|
||||
""")
|
||||
print(f'commissioning: {len(parts)}')
|
||||
|
||||
# ── 5. MORTGAGE STATS ──────────────────────────────────────────────────
|
||||
d = load_json('mortgage_stats', 'mortgage_dashboard_general')
|
||||
if d and d.get('data'):
|
||||
m = d['data']
|
||||
psql(f"""
|
||||
INSERT INTO domrf_mortgage_dashboard (snapshot_date,
|
||||
total_credit_count, total_credit_count_delta_pct,
|
||||
primary_credit_count, primary_credit_count_delta_pct,
|
||||
secondary_credit_count, secondary_credit_count_delta_pct,
|
||||
total_credit_amount, total_credit_amount_delta_pct,
|
||||
primary_credit_amount, primary_credit_amount_delta_pct,
|
||||
secondary_credit_amount, secondary_credit_amount_delta_pct,
|
||||
total_credit_avg_rate, total_credit_avg_rate_delta,
|
||||
primary_credit_avg_rate, primary_credit_avg_rate_delta,
|
||||
secondary_credit_avg_rate, secondary_credit_avg_rate_delta)
|
||||
VALUES ('{SNAP}',
|
||||
{integer(m.get('totalCreditCount'))}, {num(m.get('totalCreditCountDeltaPct'))},
|
||||
{integer(m.get('primaryCreditCount'))}, {num(m.get('primaryCreditCountDeltaPct'))},
|
||||
{integer(m.get('secondaryCreditCount'))}, {num(m.get('secondaryCreditCountDeltaPct'))},
|
||||
{num(m.get('totalCreditAmount'))}, {num(m.get('totalCreditAmountDeltaPct'))},
|
||||
{num(m.get('primaryCreditAmount'))}, {num(m.get('primaryCreditAmountDeltaPct'))},
|
||||
{num(m.get('secondaryCreditAmount'))}, {num(m.get('secondaryCreditAmountDeltaPct'))},
|
||||
{num(m.get('totalCreditAvgRate'))}, {num(m.get('totalCreditAvgRateDelta'))},
|
||||
{num(m.get('primaryCreditAvgRate'))}, {num(m.get('primaryCreditAvgRateDelta'))},
|
||||
{num(m.get('secondaryCreditAvgRate'))}, {num(m.get('secondaryCreditAvgRateDelta'))})
|
||||
ON CONFLICT (snapshot_date) DO UPDATE SET
|
||||
total_credit_count = EXCLUDED.total_credit_count,
|
||||
total_credit_amount = EXCLUDED.total_credit_amount,
|
||||
total_credit_avg_rate = EXCLUDED.total_credit_avg_rate;
|
||||
""")
|
||||
print('mortgage_dashboard: 1')
|
||||
|
||||
d = load_json('mortgage_stats', 'mortgage_dashboard_details')
|
||||
if d and d.get('data'):
|
||||
rows = d['data']
|
||||
parts = [f"('{SNAP}', {esc(r.get('currency'))}, {num(r.get('creditAmountAvg'))}, "
|
||||
f"{num(r.get('creditAmountAvgDeltaPct'))}, {num(r.get('creditAvgPeriod'))}, "
|
||||
f"{num(r.get('creditAvgPeriodDeltaPct'))}, {num(r.get('creditDebtsAmount'))}, "
|
||||
f"{num(r.get('creditDebtsAmountDeltaPct'))}, {num(r.get('creditDebtsOverduePercent'))}, "
|
||||
f"{num(r.get('creditDebtsOverduePercentDelta'))})"
|
||||
for r in rows]
|
||||
if parts:
|
||||
psql(f"""
|
||||
INSERT INTO domrf_mortgage_details (snapshot_date, currency, credit_amount_avg,
|
||||
credit_amount_avg_delta_pct, credit_avg_period, credit_avg_period_delta_pct,
|
||||
credit_debts_amount, credit_debts_amount_delta_pct,
|
||||
credit_debts_overdue_percent, credit_debts_overdue_percent_delta)
|
||||
VALUES {','.join(parts)}
|
||||
ON CONFLICT (snapshot_date, currency) DO UPDATE SET
|
||||
credit_amount_avg = EXCLUDED.credit_amount_avg,
|
||||
credit_avg_period = EXCLUDED.credit_avg_period,
|
||||
credit_debts_amount = EXCLUDED.credit_debts_amount;
|
||||
""")
|
||||
print(f'mortgage_details: {len(parts)}')
|
||||
|
||||
print('\n--- summary ---')
|
||||
out = psql(f"""
|
||||
SELECT 'launch_top' tab, COUNT(*) FROM domrf_launch_top WHERE snapshot_date = '{SNAP}'
|
||||
UNION ALL SELECT 'launch_obj_class', COUNT(*) FROM domrf_launch_obj_class WHERE snapshot_date = '{SNAP}'
|
||||
UNION ALL SELECT 'launch_monthly', COUNT(*) FROM domrf_launch_monthly WHERE snapshot_date = '{SNAP}'
|
||||
UNION ALL SELECT 'sold_ready_index', COUNT(*) FROM domrf_sold_ready_index WHERE snapshot_date = '{SNAP}'
|
||||
UNION ALL SELECT 'sold_ready_breakdown', COUNT(*) FROM domrf_sold_ready_breakdown WHERE snapshot_date = '{SNAP}'
|
||||
UNION ALL SELECT 'sold_ready_dynamics', COUNT(*) FROM domrf_sold_ready_dynamics WHERE snapshot_date = '{SNAP}'
|
||||
UNION ALL SELECT 'project_finance', COUNT(*) FROM domrf_project_finance WHERE snapshot_date = '{SNAP}'
|
||||
UNION ALL SELECT 'commissioning', COUNT(*) FROM domrf_commissioning WHERE snapshot_date = '{SNAP}'
|
||||
UNION ALL SELECT 'mortgage_dashboard', COUNT(*) FROM domrf_mortgage_dashboard WHERE snapshot_date = '{SNAP}'
|
||||
UNION ALL SELECT 'mortgage_details', COUNT(*) FROM domrf_mortgage_details WHERE snapshot_date = '{SNAP}'
|
||||
ORDER BY 1;
|
||||
""")
|
||||
print(out)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
99
data/sql/32_load_domrf_raw.py
Normal file
99
data/sql/32_load_domrf_raw.py
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
"""Universal loader: dumps every scraped JSON from data/raw/domrf_full/ into Postgres
|
||||
as JSONB rows in domrf_raw_endpoints. Idempotent (UPSERT by snapshot+section+endpoint).
|
||||
|
||||
Usage:
|
||||
PG_HOST=localhost PG_PORT=15432 PGPASSWORD=... python data/sql/32_load_domrf_raw.py
|
||||
"""
|
||||
import os, json, subprocess, sys, datetime
|
||||
|
||||
HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
ROOT = os.path.join(HERE, '..', 'raw', 'domrf_full')
|
||||
PG_PORT = os.environ.get('PG_PORT', '5432')
|
||||
PG_USER = os.environ.get('PG_USER', 'gendesign')
|
||||
PG_DB = os.environ.get('PG_DB', 'gendesign')
|
||||
PG_PASS = os.environ.get('PGPASSWORD', '')
|
||||
SNAP = os.environ.get('SNAPSHOT_DATE', datetime.date.today().isoformat())
|
||||
|
||||
|
||||
def psql(sql):
|
||||
cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
|
||||
'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT,
|
||||
'-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet']
|
||||
res = subprocess.run(cmd, input=sql, capture_output=True, text=True, encoding='utf-8')
|
||||
if res.returncode != 0:
|
||||
print('psql FAILED:', res.stderr[-1500:], file=sys.stderr)
|
||||
raise SystemExit(res.returncode)
|
||||
return res.stdout
|
||||
|
||||
|
||||
def psql_stdin(sql, stdin_data):
|
||||
"""Use psql with COPY via stdin to load big JSONs (avoids cmdline length limits)."""
|
||||
cmd = ['docker', 'run', '--rm', '-i', '-e', f'PGPASSWORD={PG_PASS}',
|
||||
'postgres:16-alpine', 'psql', '-h', 'host.docker.internal', '-p', PG_PORT,
|
||||
'-U', PG_USER, '-d', PG_DB, '-v', 'ON_ERROR_STOP=1', '--quiet']
|
||||
res = subprocess.run(cmd, input=stdin_data, capture_output=True, text=True, encoding='utf-8')
|
||||
if res.returncode != 0:
|
||||
print('psql FAILED:', res.stderr[-1500:], file=sys.stderr)
|
||||
raise SystemExit(res.returncode)
|
||||
return res.stdout
|
||||
|
||||
|
||||
def main():
|
||||
schema = open(os.path.join(HERE, '31_schema_domrf_raw.sql'), encoding='utf-8').read()
|
||||
psql(schema)
|
||||
print('schema applied')
|
||||
|
||||
loaded = 0
|
||||
skipped = 0
|
||||
by_section = {}
|
||||
|
||||
for section in sorted(os.listdir(ROOT)):
|
||||
sec_dir = os.path.join(ROOT, section)
|
||||
if not os.path.isdir(sec_dir):
|
||||
continue
|
||||
files = [f for f in os.listdir(sec_dir) if f.endswith('.json')]
|
||||
for fname in sorted(files):
|
||||
fpath = os.path.join(sec_dir, fname)
|
||||
try:
|
||||
text = open(fpath, encoding='utf-8').read()
|
||||
json.loads(text) # validate
|
||||
except Exception as e:
|
||||
print(f' SKIP invalid JSON: {section}/{fname}: {e}')
|
||||
skipped += 1
|
||||
continue
|
||||
endpoint = fname[:-5] # strip .json
|
||||
# Use COPY ... FROM STDIN for safe binary insertion
|
||||
# Use psycopg-style escape: \\ → \\\\, replace tabs/newlines in JSON keys
|
||||
# Easier: pass JSON via stdin to a Python heredoc-style INSERT
|
||||
sql = f"""
|
||||
INSERT INTO domrf_raw_endpoints (snapshot_date, section, endpoint, source_url, payload, payload_size, fetched_at)
|
||||
VALUES ('{SNAP}', $sec${section}$sec$, $end${endpoint}$end$, NULL, $j${text}$j$::jsonb, {len(text)}, NOW())
|
||||
ON CONFLICT (snapshot_date, section, endpoint) DO UPDATE SET
|
||||
payload = EXCLUDED.payload,
|
||||
payload_size = EXCLUDED.payload_size,
|
||||
fetched_at = NOW();
|
||||
"""
|
||||
psql_stdin('', sql)
|
||||
loaded += 1
|
||||
by_section.setdefault(section, 0)
|
||||
by_section[section] += 1
|
||||
|
||||
print(f'\nLOADED {loaded} files, SKIPPED {skipped}')
|
||||
print('--- by section ---')
|
||||
for s, n in sorted(by_section.items()):
|
||||
print(f' {s:22s} {n}')
|
||||
|
||||
# Final summary from DB
|
||||
print('\n--- DB summary ---')
|
||||
out = psql(f"""
|
||||
SELECT section, COUNT(*) AS endpoints, pg_size_pretty(SUM(payload_size)::bigint) AS total
|
||||
FROM domrf_raw_endpoints
|
||||
WHERE snapshot_date = '{SNAP}'
|
||||
GROUP BY section
|
||||
ORDER BY section;
|
||||
""")
|
||||
print(out)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
287
data/sql/40_relations.sql
Normal file
287
data/sql/40_relations.sql
Normal file
|
|
@ -0,0 +1,287 @@
|
|||
-- Foreign-key constraints + helper VIEWs across DOM.RF + Rosreestr tables.
|
||||
-- Idempotent: bootstraps missing dimension rows first, then ALTER TABLE ... ADD CONSTRAINT IF NOT EXISTS.
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────
|
||||
-- 1. BOOTSTRAP DIMENSION TABLES
|
||||
-- ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- Add row 0 = РФ to domrf_regions (used as "all-Russia" placeholder by some tables)
|
||||
INSERT INTO domrf_regions (region_id, region_name, federal_district)
|
||||
VALUES (0, 'Российская Федерация', '_RF')
|
||||
ON CONFLICT (region_id) DO NOTHING;
|
||||
|
||||
-- Backfill domrf_snapshots from every snapshot_date currently used in children
|
||||
INSERT INTO domrf_snapshots (snapshot_date, notes)
|
||||
SELECT DISTINCT s, 'auto-backfilled by 40_relations.sql' FROM (
|
||||
SELECT snapshot_date AS s FROM domrf_developers_full
|
||||
UNION SELECT snapshot_date FROM domrf_escrow_banks
|
||||
UNION SELECT snapshot_date FROM domrf_guaranty_regions
|
||||
UNION SELECT snapshot_date FROM domrf_housing_summary
|
||||
UNION SELECT snapshot_date FROM domrf_planned_commissioning
|
||||
UNION SELECT snapshot_date FROM domrf_sold_out
|
||||
UNION SELECT snapshot_date FROM domrf_sold_out_by_year
|
||||
UNION SELECT snapshot_date FROM domrf_sold_out_by_progress
|
||||
UNION SELECT snapshot_date FROM domrf_mortgage_rates
|
||||
UNION SELECT snapshot_date FROM domrf_region_aggregates
|
||||
UNION SELECT snapshot_date FROM domrf_flat_area_distribution
|
||||
UNION SELECT snapshot_date FROM domrf_developer_aggregates
|
||||
UNION SELECT snapshot_date FROM domrf_launch_top
|
||||
UNION SELECT snapshot_date FROM domrf_launch_obj_class
|
||||
UNION SELECT snapshot_date FROM domrf_launch_monthly
|
||||
UNION SELECT snapshot_date FROM domrf_sold_ready_index
|
||||
UNION SELECT snapshot_date FROM domrf_sold_ready_breakdown
|
||||
UNION SELECT snapshot_date FROM domrf_sold_ready_dynamics
|
||||
UNION SELECT snapshot_date FROM domrf_project_finance
|
||||
UNION SELECT snapshot_date FROM domrf_commissioning
|
||||
UNION SELECT snapshot_date FROM domrf_mortgage_dashboard
|
||||
UNION SELECT snapshot_date FROM domrf_mortgage_details
|
||||
) AS dates
|
||||
ON CONFLICT (snapshot_date) DO NOTHING;
|
||||
|
||||
-- Helper to add a constraint only if it does not already exist
|
||||
-- (PostgreSQL ≥9.6: do via DO block; idempotent)
|
||||
DO $$
|
||||
DECLARE
|
||||
rec RECORD;
|
||||
BEGIN
|
||||
-- ──────────────────────────────────────────────────────────────────────
|
||||
-- 2. SNAPSHOT FKs (every domrf_* with snapshot_date column)
|
||||
-- ──────────────────────────────────────────────────────────────────────
|
||||
FOR rec IN SELECT * FROM (VALUES
|
||||
('domrf_developers_full', 'fk_snap_developers_full'),
|
||||
('domrf_escrow_banks', 'fk_snap_escrow_banks'),
|
||||
('domrf_guaranty_regions', 'fk_snap_guaranty_regions'),
|
||||
('domrf_housing_summary', 'fk_snap_housing_summary'),
|
||||
('domrf_planned_commissioning', 'fk_snap_planned_commissioning'),
|
||||
('domrf_sold_out', 'fk_snap_sold_out'),
|
||||
('domrf_sold_out_by_year', 'fk_snap_sold_out_by_year'),
|
||||
('domrf_sold_out_by_progress', 'fk_snap_sold_out_by_progress'),
|
||||
('domrf_mortgage_rates', 'fk_snap_mortgage_rates'),
|
||||
('domrf_region_aggregates', 'fk_snap_region_aggregates'),
|
||||
('domrf_flat_area_distribution', 'fk_snap_flat_area_distribution'),
|
||||
('domrf_developer_aggregates', 'fk_snap_developer_aggregates'),
|
||||
('domrf_launch_top', 'fk_snap_launch_top'),
|
||||
('domrf_launch_obj_class', 'fk_snap_launch_obj_class'),
|
||||
('domrf_launch_monthly', 'fk_snap_launch_monthly'),
|
||||
('domrf_sold_ready_index', 'fk_snap_sold_ready_index'),
|
||||
('domrf_sold_ready_breakdown', 'fk_snap_sold_ready_breakdown'),
|
||||
('domrf_sold_ready_dynamics', 'fk_snap_sold_ready_dynamics'),
|
||||
('domrf_project_finance', 'fk_snap_project_finance'),
|
||||
('domrf_commissioning', 'fk_snap_commissioning'),
|
||||
('domrf_mortgage_dashboard', 'fk_snap_mortgage_dashboard'),
|
||||
('domrf_mortgage_details', 'fk_snap_mortgage_details'),
|
||||
('domrf_raw_endpoints', 'fk_snap_raw_endpoints')
|
||||
) AS t(tbl, cn)
|
||||
LOOP
|
||||
IF NOT EXISTS (
|
||||
SELECT 1 FROM pg_constraint WHERE conname = rec.cn
|
||||
) THEN
|
||||
EXECUTE format(
|
||||
'ALTER TABLE %I ADD CONSTRAINT %I FOREIGN KEY (snapshot_date) REFERENCES domrf_snapshots(snapshot_date) ON UPDATE CASCADE',
|
||||
rec.tbl, rec.cn
|
||||
);
|
||||
END IF;
|
||||
END LOOP;
|
||||
|
||||
-- ──────────────────────────────────────────────────────────────────────
|
||||
-- 3. REGION FKs (domrf_region_aggregates / flat_area_distribution / developer_aggregates → domrf_regions)
|
||||
-- ──────────────────────────────────────────────────────────────────────
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_region_aggregates_region') THEN
|
||||
ALTER TABLE domrf_region_aggregates
|
||||
ADD CONSTRAINT fk_region_aggregates_region
|
||||
FOREIGN KEY (region_id) REFERENCES domrf_regions(region_id) ON UPDATE CASCADE;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_flat_area_region') THEN
|
||||
ALTER TABLE domrf_flat_area_distribution
|
||||
ADD CONSTRAINT fk_flat_area_region
|
||||
FOREIGN KEY (region_id) REFERENCES domrf_regions(region_id) ON UPDATE CASCADE;
|
||||
END IF;
|
||||
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_developer_agg_region') THEN
|
||||
ALTER TABLE domrf_developer_aggregates
|
||||
ADD CONSTRAINT fk_developer_agg_region
|
||||
FOREIGN KEY (region_id) REFERENCES domrf_regions(region_id) ON UPDATE CASCADE;
|
||||
END IF;
|
||||
|
||||
-- ──────────────────────────────────────────────────────────────────────
|
||||
-- 4. DEVELOPER FK (domrf_developer_aggregates → domrf_developers)
|
||||
-- ──────────────────────────────────────────────────────────────────────
|
||||
IF NOT EXISTS (SELECT 1 FROM pg_constraint WHERE conname = 'fk_developer_agg_developer') THEN
|
||||
ALTER TABLE domrf_developer_aggregates
|
||||
ADD CONSTRAINT fk_developer_agg_developer
|
||||
FOREIGN KEY (developer_id) REFERENCES domrf_developers(developer_id) ON UPDATE CASCADE;
|
||||
END IF;
|
||||
END $$;
|
||||
|
||||
-- ─────────────────────────────────────────────────────────────────────────
|
||||
-- 5. HELPER VIEWS
|
||||
-- ─────────────────────────────────────────────────────────────────────────
|
||||
|
||||
-- v_region_master: одна строка на регион — все ключевые метрики на последний снапшот
|
||||
DROP VIEW IF EXISTS v_region_master CASCADE;
|
||||
CREATE VIEW v_region_master AS
|
||||
WITH latest AS (SELECT MAX(snapshot_date) AS d FROM domrf_snapshots)
|
||||
SELECT
|
||||
r.region_id,
|
||||
r.region_name,
|
||||
r.federal_district,
|
||||
-- Quartirografia totals (TOTAL row)
|
||||
rta.flat_count AS total_flats,
|
||||
rta.area_sqm AS total_area_sqm,
|
||||
-- Sold-out (DOM.RF PDF data)
|
||||
so.total_area_th_sqm AS sold_out_total_th_sqm,
|
||||
so.sold_pct AS sold_out_sold_pct,
|
||||
so.unsold_pct AS sold_out_unsold_pct,
|
||||
-- Sold-ready breakdown (Playwright API data)
|
||||
srb.square_sum AS sold_ready_square,
|
||||
srb.sold_perc AS sold_ready_sold_pct,
|
||||
srb.ready_perc AS sold_ready_ready_pct,
|
||||
srb.sold_ready_perc AS sold_ready_balance_pct,
|
||||
-- Project finance
|
||||
pf.rns_cnt AS project_finance_permits,
|
||||
pf.dev_cnt AS project_finance_developers,
|
||||
pf.guaranty_escrow_dev_cnt AS project_finance_escrow_devs,
|
||||
-- Guaranty (XLSX)
|
||||
g.fz214_developers AS guaranty_fz214_devs,
|
||||
g.escrow_developers AS guaranty_escrow_devs,
|
||||
-- Launch (запуски новых)
|
||||
(SELECT value FROM domrf_launch_top
|
||||
WHERE dim_type = 'region' AND metric_type = 'app'
|
||||
AND entity_id = r.region_id::text AND snapshot_date = (SELECT d FROM latest)
|
||||
LIMIT 1) AS launch_app_2026_sqm,
|
||||
(SELECT value FROM domrf_launch_top
|
||||
WHERE dim_type = 'region' AND metric_type = 'rnv'
|
||||
AND entity_id = r.region_id::text AND snapshot_date = (SELECT d FROM latest)
|
||||
LIMIT 1) AS launch_rnv_2026_sqm
|
||||
FROM domrf_regions r
|
||||
LEFT JOIN domrf_region_aggregates rta
|
||||
ON rta.region_id = r.region_id
|
||||
AND rta.room_count_type = 'TOTAL'
|
||||
AND rta.snapshot_date = (SELECT d FROM latest)
|
||||
LEFT JOIN domrf_sold_out so
|
||||
ON so.territory_name = r.region_name
|
||||
AND so.snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_sold_out WHERE territory_name = r.region_name)
|
||||
LEFT JOIN domrf_sold_ready_breakdown srb
|
||||
ON srb.entity_key = r.region_name
|
||||
AND srb.chart_type = 'regionChart'
|
||||
AND srb.snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_sold_ready_breakdown WHERE chart_type='regionChart')
|
||||
LEFT JOIN domrf_project_finance pf
|
||||
ON pf.subject_name = r.region_name
|
||||
AND pf.snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_project_finance)
|
||||
LEFT JOIN domrf_guaranty_regions g
|
||||
ON g.territory_name = r.region_name
|
||||
AND g.snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_guaranty_regions)
|
||||
WHERE r.region_id > 0
|
||||
ORDER BY total_area_sqm DESC NULLS LAST;
|
||||
|
||||
COMMENT ON VIEW v_region_master IS
|
||||
'Один срез по региону: квартирография + распроданность (DOM.РФ + sold_out PDF) + проектное финансирование + запуски/ввод 2026';
|
||||
|
||||
-- v_developer_master: профиль застройщика (latest snapshot)
|
||||
DROP VIEW IF EXISTS v_developer_master CASCADE;
|
||||
CREATE VIEW v_developer_master AS
|
||||
WITH latest AS (SELECT MAX(snapshot_date) AS d FROM domrf_snapshots)
|
||||
SELECT
|
||||
d.developer_id,
|
||||
d.developer_name,
|
||||
-- Group-of-companies aggregates (from JSON dashboard)
|
||||
dat.flat_count AS group_flat_count,
|
||||
dat.area_sqm AS group_area_sqm,
|
||||
-- Per-юрлицо реестр (XLSX)
|
||||
df.area_thousand_sqm AS legal_area_th_sqm,
|
||||
df.permits_count AS legal_permits,
|
||||
df.houses_count AS legal_houses,
|
||||
df.flats_count AS legal_flats,
|
||||
df.market_share_pct AS market_share_pct,
|
||||
-- Запуски / ввод 2026 (из launch_top, dim=developer)
|
||||
(SELECT value FROM domrf_launch_top
|
||||
WHERE dim_type='developer' AND metric_type='app'
|
||||
AND entity_id = d.developer_id AND snapshot_date = (SELECT d FROM latest)
|
||||
LIMIT 1) AS launch_app_2026_sqm,
|
||||
(SELECT value FROM domrf_launch_top
|
||||
WHERE dim_type='developer' AND metric_type='rnv'
|
||||
AND entity_id = d.developer_id AND snapshot_date = (SELECT d FROM latest)
|
||||
LIMIT 1) AS launch_rnv_2026_sqm,
|
||||
-- Распроданность (devChart)
|
||||
(SELECT square_sum FROM domrf_sold_ready_breakdown
|
||||
WHERE chart_type='devChart' AND entity_key = d.developer_name
|
||||
AND snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_sold_ready_breakdown WHERE chart_type='devChart')
|
||||
LIMIT 1) AS sold_ready_square,
|
||||
(SELECT sold_perc FROM domrf_sold_ready_breakdown
|
||||
WHERE chart_type='devChart' AND entity_key = d.developer_name
|
||||
AND snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_sold_ready_breakdown WHERE chart_type='devChart')
|
||||
LIMIT 1) AS sold_ready_sold_pct
|
||||
FROM domrf_developers d
|
||||
LEFT JOIN domrf_developer_aggregates dat
|
||||
ON dat.developer_id = d.developer_id
|
||||
AND dat.room_count_type = 'TOTAL'
|
||||
AND dat.snapshot_date = (SELECT d FROM latest)
|
||||
LEFT JOIN domrf_developers_full df
|
||||
ON df.developer_name = d.developer_name
|
||||
AND df.snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_developers_full WHERE developer_name = d.developer_name);
|
||||
|
||||
COMMENT ON VIEW v_developer_master IS
|
||||
'Профиль застройщика: реестр + квартирография по комнатностям + запуски/ввод + распроданность';
|
||||
|
||||
-- v_rosreestr_market_by_region — срез цен сделок и застройщиков по региону
|
||||
DROP VIEW IF EXISTS v_rosreestr_market_by_region CASCADE;
|
||||
CREATE VIEW v_rosreestr_market_by_region AS
|
||||
SELECT
|
||||
r.region_id,
|
||||
r.region_name,
|
||||
r.federal_district,
|
||||
rar.period_start_date,
|
||||
rar.doc_type,
|
||||
rar.deals_count,
|
||||
rar.median_price_sqm AS median_rub_sqm,
|
||||
rar.p10_price_sqm AS p10_rub_sqm,
|
||||
rar.p90_price_sqm AS p90_rub_sqm
|
||||
FROM domrf_regions r
|
||||
JOIN rr_agg_region rar
|
||||
ON rar.region_code = r.region_id
|
||||
WHERE rar.realestate_type_code = '002001003000' -- Помещения (квартиры)
|
||||
ORDER BY r.region_id, rar.period_start_date, rar.doc_type;
|
||||
|
||||
COMMENT ON VIEW v_rosreestr_market_by_region IS
|
||||
'Цены сделок Росреестра по регионам с DOM.РФ-именами регионов (квартиры)';
|
||||
|
||||
-- v_market_pulse_rf — РФ-уровень summary с разных источников за last snapshot
|
||||
DROP VIEW IF EXISTS v_market_pulse_rf CASCADE;
|
||||
CREATE VIEW v_market_pulse_rf AS
|
||||
SELECT
|
||||
-- Quartirografia
|
||||
(SELECT SUM(area_sqm)/1e6 FROM domrf_region_aggregates
|
||||
WHERE room_count_type='TOTAL'
|
||||
AND snapshot_date = (SELECT MAX(snapshot_date) FROM domrf_region_aggregates)) AS total_construction_mln_sqm,
|
||||
-- Sold-ready
|
||||
(SELECT square_sum/1e6 FROM domrf_sold_ready_index
|
||||
ORDER BY snapshot_date DESC LIMIT 1) AS sold_ready_total_mln_sqm,
|
||||
(SELECT sold_perc FROM domrf_sold_ready_index
|
||||
ORDER BY snapshot_date DESC LIMIT 1) AS sold_pct,
|
||||
(SELECT ready_perc FROM domrf_sold_ready_index
|
||||
ORDER BY snapshot_date DESC LIMIT 1) AS ready_pct,
|
||||
-- Прайс
|
||||
(SELECT avg_price_per_sqm FROM domrf_sold_out
|
||||
WHERE territory_name = 'Российская Федерация'
|
||||
ORDER BY snapshot_date DESC LIMIT 1) AS rf_avg_price_per_sqm,
|
||||
-- Ставки
|
||||
(SELECT primary_rate FROM domrf_mortgage_rates
|
||||
WHERE bank_name = 'Средневзвешенная'
|
||||
ORDER BY snapshot_date DESC LIMIT 1) AS market_primary_rate,
|
||||
(SELECT secondary_rate FROM domrf_mortgage_rates
|
||||
WHERE bank_name = 'Средневзвешенная'
|
||||
ORDER BY snapshot_date DESC LIMIT 1) AS market_secondary_rate,
|
||||
-- Реальные ставки (по выданным кредитам)
|
||||
(SELECT primary_credit_avg_rate FROM domrf_mortgage_dashboard
|
||||
ORDER BY snapshot_date DESC LIMIT 1) AS actual_primary_rate,
|
||||
(SELECT secondary_credit_avg_rate FROM domrf_mortgage_dashboard
|
||||
ORDER BY snapshot_date DESC LIMIT 1) AS actual_secondary_rate,
|
||||
-- Объём кредитования
|
||||
(SELECT total_credit_count FROM domrf_mortgage_dashboard
|
||||
ORDER BY snapshot_date DESC LIMIT 1) AS total_credit_count,
|
||||
(SELECT total_credit_amount/1e9 FROM domrf_mortgage_dashboard
|
||||
ORDER BY snapshot_date DESC LIMIT 1) AS total_credit_amount_bln_rub;
|
||||
|
||||
COMMENT ON VIEW v_market_pulse_rf IS
|
||||
'РФ pulse: объём строительства, распроданность, средняя цена, ставки (рыночные vs реальные), объём ипотеки';
|
||||
11
data/sql/decode_to_file.py
Normal file
11
data/sql/decode_to_file.py
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
"""Reads JSON-encoded string from stdin (the kind MCP returns when wrapping a string),
|
||||
and writes the decoded value as UTF-8 to the given file."""
|
||||
import json, sys, os
|
||||
out_path = sys.argv[1]
|
||||
raw = sys.stdin.read()
|
||||
# raw is a JSON string like "[{\"name\":\"...\"}]" — already JSON-quoted
|
||||
decoded = json.loads(raw)
|
||||
os.makedirs(os.path.dirname(out_path) or '.', exist_ok=True)
|
||||
with open(out_path, 'w', encoding='utf-8') as f:
|
||||
f.write(decoded)
|
||||
print(f'wrote {out_path} ({len(decoded)} chars)')
|
||||
57
data/sql/upload_server.py
Normal file
57
data/sql/upload_server.py
Normal file
|
|
@ -0,0 +1,57 @@
|
|||
"""Tiny HTTP receiver: POST <body> to /<path>, file gets written to data/raw/domrf/<path>.
|
||||
|
||||
Usage:
|
||||
python data/sql/upload_server.py # listens on 127.0.0.1:8765
|
||||
|
||||
In browser DevTools:
|
||||
fetch('http://127.0.0.1:8765/regions.json', { method: 'POST', body: JSON.stringify(window.__domrf['regions']) });
|
||||
"""
|
||||
from http.server import BaseHTTPRequestHandler, HTTPServer
|
||||
import os
|
||||
import sys
|
||||
|
||||
ROOT = os.path.join(os.path.dirname(__file__), '..', 'raw', 'domrf')
|
||||
os.makedirs(ROOT, exist_ok=True)
|
||||
|
||||
|
||||
class H(BaseHTTPRequestHandler):
|
||||
def _cors(self):
|
||||
self.send_header('Access-Control-Allow-Origin', '*')
|
||||
self.send_header('Access-Control-Allow-Methods', 'POST, OPTIONS')
|
||||
self.send_header('Access-Control-Allow-Headers', 'Content-Type')
|
||||
|
||||
def do_OPTIONS(self):
|
||||
self.send_response(204)
|
||||
self._cors()
|
||||
self.end_headers()
|
||||
|
||||
def do_POST(self):
|
||||
try:
|
||||
length = int(self.headers.get('Content-Length', 0))
|
||||
body = self.rfile.read(length)
|
||||
name = self.path.lstrip('/').replace('..', '_').replace('\\', '_')
|
||||
if not name:
|
||||
name = 'upload.json'
|
||||
path = os.path.join(ROOT, name)
|
||||
with open(path, 'wb') as f:
|
||||
f.write(body)
|
||||
print(f'WROTE {path} ({len(body)} bytes)', flush=True)
|
||||
self.send_response(200)
|
||||
self._cors()
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.end_headers()
|
||||
self.wfile.write(b'{"ok":true}')
|
||||
except Exception as e:
|
||||
self.send_response(500)
|
||||
self._cors()
|
||||
self.end_headers()
|
||||
self.wfile.write(str(e).encode())
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
addr = ('127.0.0.1', 8765)
|
||||
print(f'listening on http://{addr[0]}:{addr[1]}, output dir: {os.path.abspath(ROOT)}', flush=True)
|
||||
HTTPServer(addr, H).serve_forever()
|
||||
File diff suppressed because one or more lines are too long
24
sshkey.txt
24
sshkey.txt
|
|
@ -56,3 +56,27 @@ Port: 15432
|
|||
Database: gendesign
|
||||
Username: gendesign
|
||||
Password: 2J2SBPMKuS998fiwhtQqDhMI
|
||||
|
||||
Локально установить pre-commit один раз:
|
||||
make pre-commit-install
|
||||
# или:
|
||||
pip install pre-commit
|
||||
pre-commit install
|
||||
|
||||
python -m pre_commit install
|
||||
|
||||
python -m pre_commit run --all-files
|
||||
# или, если PATH добавлен:
|
||||
pre-commit run --all-files
|
||||
|
||||
Только текущая сессия:
|
||||
|
||||
$env:PATH += ";$env:LOCALAPPDATA\Python\pythoncore-3.14-64\Scripts"
|
||||
pre-commit install
|
||||
Постоянно (нужно ОДИН раз):
|
||||
|
||||
[Environment]::SetEnvironmentVariable(
|
||||
"PATH",
|
||||
$env:PATH + ";$env:LOCALAPPDATA\Python\pythoncore-3.14-64\Scripts",
|
||||
"User"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue