7b32edc34d
Per code review audit (May 14): BRIN index на partitioned parent table dead weight. PG16 НЕ propagates такие индексы на child partitions; pg_stat показывает idx_scan=0 → никогда не использовался. ## Why dead rosreestr_deals — yearly partitioned по period_start_date. Queries делают partition pruning через partition key. BRIN index на parent существует, но: 1. На child partitions его нет (PG16 не reflects) 2. Parent сам не имеет данных (partitioned out) 3. Queries никогда не trigger BRIN scan Maintenance overhead (storage, autoanalyze) без value. ## Migration `data/sql/89_drop_dead_brin_rosreestr_deals.sql` — атомарный DROP с idempotent `IF EXISTS`. Apply order: после 88_*. ## Cleanup `data/sql/01_schema_rosreestr_deals.sql:65-67` — закомментировал оригинальное CREATE с note: fresh DB bootstrap НЕ создаст index заново. ## Rollback path Если в будущем нужен range scan optimization — re-create per-partition: ```sql CREATE INDEX rosreestr_deals_2024q1_period_brin ON rosreestr_deals_2024q1 USING BRIN (period_start_date); -- per-partition × N partitions ``` Но обычно partition pruning достаточно. Измерь EXPLAIN перед re-creation. ## Vault `fixes/Migration_Drop_Dead_Brin_May14.md` создан (status: ready_to_apply). Apply на prod: `psql ... -f data/sql/89_drop_dead_brin_rosreestr_deals.sql` после merge. Co-authored-by: lekss361 <claudestars@proton.me>
95 lines
5.6 KiB
SQL
95 lines
5.6 KiB
SQL
-- Rosreestr open dataset: dataset_СДЕЛКИ (registered real estate deals).
|
||
-- Source: https://rosreestr.gov.ru/data-sets/{Q}%20квартал%20{YYYY}г./dataset_СДЕЛКИ_r-r_01-92_y_{YYYY}_q_{Q}.csv.zip
|
||
-- Encoding: UTF-8. Separator: '~' (since 2024 Q4) or ';' (2024 Q3 only).
|
||
-- Granularity: aggregated rows — same (region+district+city+cad_quarter+street+type+material+year+floor+purpose+area+doc_type) deals
|
||
-- collapsed into a single row with `deal_count` (was `number`) and AVERAGE price.
|
||
-- Time granularity: quarter (period_start_date = first day of quarter).
|
||
-- Coverage: ALL registered transactions (DKP=secondary+land+commercial, DDU=primary in МКД).
|
||
-- Excluded: gratuitous deals (gift/inheritance), buyer/seller PII, exact deal date, apartment number, room count, mortgage flag.
|
||
-- Data quality notes (observed in 7 loaded quarters):
|
||
-- * deal_price может быть мусорным с десятками знаков — фильтруем при INSERT
|
||
-- * year_build бывает с датой типа '09.07.1988' — оставляем NULL если не 4-цифровой год
|
||
-- * wall_material_code/realestate_type_code иногда '"code1;code2"' — поэтому TEXT вместо VARCHAR(15)
|
||
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals (
|
||
id BIGSERIAL,
|
||
-- Origin
|
||
source_quarter VARCHAR(8) NOT NULL, -- '2024Q3' for traceability
|
||
period_start_date DATE NOT NULL, -- first day of the quarter (partition key)
|
||
-- Geo
|
||
region_code SMALLINT NOT NULL, -- 66 = Свердловская обл, 77 = Москва, ...
|
||
okato TEXT, -- ОКАТО (often '-' or 11 digits)
|
||
district TEXT, -- муниципальный район/округ
|
||
city TEXT, -- населённый пункт (часто пустой)
|
||
quarter_cad_number VARCHAR(30) NOT NULL, -- '66:41:0506001' — гео-ключ для JOIN с ПКК / NSPD
|
||
street TEXT,
|
||
-- Object
|
||
realestate_type_code TEXT, -- 002001001000=ЗУ | 002001002000=Здание | 002001003000=Помещение | 002001009000=Машиноместо
|
||
wall_material_code TEXT, -- справочник 126_УНСИ (может быть multi-value)
|
||
purpose_code TEXT, -- 66/67/148_УНСИ
|
||
year_build SMALLINT,
|
||
floor TEXT,
|
||
area NUMERIC, -- м²
|
||
-- Deal
|
||
doc_type VARCHAR(8) NOT NULL, -- 'ДКП' | 'ДДУ'
|
||
deal_price NUMERIC, -- средняя если deal_count > 1
|
||
currency VARCHAR(20) DEFAULT 'рубль',
|
||
deal_count INT NOT NULL DEFAULT 1, -- было `number`
|
||
-- Computed
|
||
price_per_sqm NUMERIC GENERATED ALWAYS AS (
|
||
CASE WHEN area > 0 AND deal_price > 0 THEN deal_price / area ELSE NULL END
|
||
) STORED,
|
||
-- Audit
|
||
loaded_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
|
||
PRIMARY KEY (id, period_start_date)
|
||
) PARTITION BY RANGE (period_start_date);
|
||
|
||
-- Quarterly partitions for available data (2024 Q3 — 2026 Q1)
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2024q3 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2024-07-01') TO ('2024-10-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2024q4 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2024-10-01') TO ('2025-01-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q1 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2025-01-01') TO ('2025-04-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q2 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2025-04-01') TO ('2025-07-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q3 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2025-07-01') TO ('2025-10-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2025q4 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2025-10-01') TO ('2026-01-01');
|
||
CREATE TABLE IF NOT EXISTS rosreestr_deals_2026q1 PARTITION OF rosreestr_deals
|
||
FOR VALUES FROM ('2026-01-01') TO ('2026-04-01');
|
||
|
||
-- Indexes
|
||
CREATE INDEX IF NOT EXISTS rosreestr_deals_region_cq_idx
|
||
ON rosreestr_deals (region_code, quarter_cad_number);
|
||
-- DROPPED 2026-05-14: BRIN on partitioned parent is dead weight — PG16 does
|
||
-- not propagate it to child partitions and idx_scan=0 confirmed zero usage.
|
||
-- See data/sql/89_drop_dead_brin_rosreestr_deals.sql for the removal migration.
|
||
-- CREATE INDEX IF NOT EXISTS rosreestr_deals_period_brin
|
||
-- ON rosreestr_deals USING BRIN (period_start_date);
|
||
CREATE INDEX IF NOT EXISTS rosreestr_deals_dealtype_idx
|
||
ON rosreestr_deals (doc_type) WHERE doc_type IN ('ДКП', 'ДДУ');
|
||
CREATE INDEX IF NOT EXISTS rosreestr_deals_realtype_idx
|
||
ON rosreestr_deals (realestate_type_code);
|
||
|
||
-- Staging: один общий unlogged-буфер. Перед каждым кварталом TRUNCATE.
|
||
CREATE UNLOGGED TABLE IF NOT EXISTS rosreestr_deals_staging (
|
||
deal_count INT,
|
||
okato TEXT,
|
||
region_code SMALLINT,
|
||
district TEXT,
|
||
city TEXT,
|
||
quarter_cad_number TEXT,
|
||
street TEXT,
|
||
realestate_type_code TEXT,
|
||
wall_material_code TEXT,
|
||
year_build TEXT,
|
||
floor TEXT,
|
||
purpose_code TEXT,
|
||
area NUMERIC,
|
||
period_start_date DATE,
|
||
deal_price NUMERIC,
|
||
currency VARCHAR(20),
|
||
doc_type VARCHAR(8)
|
||
);
|