92 lines
5.3 KiB
SQL
92 lines
5.3 KiB
SQL
-- Rosreestr open dataset: dataset_СДЕЛКИ (registered real estate deals).
|
||
-- Source: https://rosreestr.gov.ru/data-sets/{Q}%20квартал%20{YYYY}г./dataset_СДЕЛКИ_r-r_01-92_y_{YYYY}_q_{Q}.csv.zip
|
||
-- Encoding: UTF-8. Separator: '~' (since 2024 Q4) or ';' (2024 Q3 only).
|
||
-- Granularity: aggregated rows — same (region+district+city+cad_quarter+street+type+material+year+floor+purpose+area+doc_type) deals
|
||
-- collapsed into a single row with `deal_count` (was `number`) and AVERAGE price.
|
||
-- Time granularity: quarter (period_start_date = first day of quarter).
|
||
-- Coverage: ALL registered transactions (DKP=secondary+land+commercial, DDU=primary in МКД).
|
||
-- Excluded: gratuitous deals (gift/inheritance), buyer/seller PII, exact deal date, apartment number, room count, mortgage flag.
|
||
-- Data quality notes (observed in 7 loaded quarters):
|
||
-- * deal_price может быть мусорным с десятками знаков — фильтруем при INSERT
|
||
-- * year_build бывает с датой типа '09.07.1988' — оставляем NULL если не 4-цифровой год
|
||
-- * wall_material_code/realestate_type_code иногда '"code1;code2"' — поэтому TEXT вместо VARCHAR(15)
|
||
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals (
|
||
id BIGSERIAL,
|
||
-- Origin
|
||
source_quarter VARCHAR(8) NOT NULL, -- '2024Q3' for traceability
|
||
period_start_date DATE NOT NULL, -- first day of the quarter (partition key)
|
||
-- Geo
|
||
region_code SMALLINT NOT NULL, -- 66 = Свердловская обл, 77 = Москва, ...
|
||
okato TEXT, -- ОКАТО (often '-' or 11 digits)
|
||
district TEXT, -- муниципальный район/округ
|
||
city TEXT, -- населённый пункт (часто пустой)
|
||
quarter_cad_number VARCHAR(30) NOT NULL, -- '66:41:0506001' — гео-ключ для JOIN с ПКК / NSPD
|
||
street TEXT,
|
||
-- Object
|
||
realestate_type_code TEXT, -- 002001001000=ЗУ | 002001002000=Здание | 002001003000=Помещение | 002001009000=Машиноместо
|
||
wall_material_code TEXT, -- справочник 126_УНСИ (может быть multi-value)
|
||
purpose_code TEXT, -- 66/67/148_УНСИ
|
||
year_build SMALLINT,
|
||
floor TEXT,
|
||
area NUMERIC, -- м²
|
||
-- Deal
|
||
doc_type VARCHAR(8) NOT NULL, -- 'ДКП' | 'ДДУ'
|
||
deal_price NUMERIC, -- средняя если deal_count > 1
|
||
currency VARCHAR(20) DEFAULT 'рубль',
|
||
deal_count INT NOT NULL DEFAULT 1, -- было `number`
|
||
-- Computed
|
||
price_per_sqm NUMERIC GENERATED ALWAYS AS (
|
||
CASE WHEN area > 0 AND deal_price > 0 THEN deal_price / area ELSE NULL END
|
||
) STORED,
|
||
-- Audit
|
||
loaded_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||
PRIMARY KEY (id, period_start_date)
|
||
) PARTITION BY RANGE (period_start_date);
|
||
|
||
-- Quarterly partitions for available data (2024 Q3 — 2026 Q1)
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2024q3 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2024-07-01') TO ('2024-10-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2024q4 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2024-10-01') TO ('2025-01-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q1 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2025-01-01') TO ('2025-04-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q2 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2025-04-01') TO ('2025-07-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q3 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2025-07-01') TO ('2025-10-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q4 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2025-10-01') TO ('2026-01-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2026q1 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2026-01-01') TO ('2026-04-01');
|
||
|
||
-- Indexes
|
||
CREATE INDEX IF NOT EXISTS rosreestr_deals_region_cq_idx
|
||
ON rosreestr_deals (region_code, quarter_cad_number);
|
||
CREATE INDEX IF NOT EXISTS rosreestr_deals_period_brin
|
||
ON rosreestr_deals USING BRIN (period_start_date);
|
||
CREATE INDEX IF NOT EXISTS rosreestr_deals_dealtype_idx
|
||
ON rosreestr_deals (doc_type) WHERE doc_type IN ('ДКП', 'ДДУ');
|
||
CREATE INDEX IF NOT EXISTS rosreestr_deals_realtype_idx
|
||
ON rosreestr_deals (realestate_type_code);
|
||
|
||
-- Staging: один общий unlogged-буфер. Перед каждым кварталом TRUNCATE.
|
||
CREATE UNLOGGED TABLE IF NOT EXISTS rosreestr_deals_staging (
|
||
deal_count INT,
|
||
okato TEXT,
|
||
region_code SMALLINT,
|
||
district TEXT,
|
||
city TEXT,
|
||
quarter_cad_number TEXT,
|
||
street TEXT,
|
||
realestate_type_code TEXT,
|
||
wall_material_code TEXT,
|
||
year_build TEXT,
|
||
floor TEXT,
|
||
purpose_code TEXT,
|
||
area NUMERIC,
|
||
period_start_date DATE,
|
||
deal_price NUMERIC,
|
||
currency VARCHAR(20),
|
||
doc_type VARCHAR(8)
|
||
);
|