From 697047d17d27350fe33a54af46e3be471cb1766e Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 15:26:41 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20SQL=20migration=20029=20?= =?UTF-8?q?=E2=80=94=20extend=20matching/valuation/dynamics=20tables=20(PR?= =?UTF-8?q?=20#445=20follow-up)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses 3 MEDIUM follow-ups from deep-code-reviewer on PR #445: A. house_sources: add ext_url, raw_payload (GIN), last_seen_at — Stage 8 conflict resolution B. listing_sources: add price_rub, area_m2, floor, rooms_count, raw_payload (GIN), last_seen_at C. external_valuations: add sale/rent split columns, chart_change_pct/dir, filters_hash, external_house_id D. houses_price_dynamics: add room_count/prices_type/period dims + replace 3-col UNIQUE with 6-col E. houses: add cadastral_number (BTree idx) for Tier-0 Cian Valuation house matching All ADD COLUMN IF NOT EXISTS. DO $$ block guards UNIQUE constraint replacement idempotently. --- ...029_extend_matching_valuation_dynamics.sql | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) create mode 100644 tradein-mvp/backend/data/sql/029_extend_matching_valuation_dynamics.sql diff --git a/tradein-mvp/backend/data/sql/029_extend_matching_valuation_dynamics.sql b/tradein-mvp/backend/data/sql/029_extend_matching_valuation_dynamics.sql new file mode 100644 index 00000000..657dd7bc --- /dev/null +++ b/tradein-mvp/backend/data/sql/029_extend_matching_valuation_dynamics.sql @@ -0,0 +1,184 @@ +-- 029_extend_matching_valuation_dynamics.sql +-- Purpose: Extend matching/valuation/dynamics tables per PR #445 deep-code-reviewer follow-ups. +-- Addresses 3 MEDIUM blockers: +-- A. house_sources — add ext_url / raw_payload / last_seen_at (Stage 8 conflict resolution) +-- B. listing_sources — add price_rub / area_m2 / floor / rooms_count / raw_payload / last_seen_at +-- C. external_valuations — split sale/rent + chart_change_pct/dir + filters_hash + external_house_id +-- (Stage 7 Cian Valuation full save) +-- D. houses_price_dynamics — add room_count / prices_type / period dimensions + replace UNIQUE +-- (Stage 6 newbuilding multi-dim chart) +-- E. houses — add cadastral_number for Tier-0 Cian Valuation house matching +-- +-- Dependencies (must already exist): +-- - house_sources (028_matching_tables.sql) +-- - listing_sources (028_matching_tables.sql) +-- - external_valuations (026_external_valuations.sql) +-- - houses_price_dynamics (024_houses_price_dynamics.sql) +-- - houses (009_houses.sql) +-- +-- Deploy order: Apply after 028. +-- Re-run safe: all ADD COLUMN IF NOT EXISTS; DO $$ blocks guard constraint changes. +-- +-- NOTE on column naming: house_sources uses (ext_source, ext_id) per 028 DDL. +-- listing_sources uses (ext_source, ext_id, source_url, last_scraped_at) per 028 DDL. +-- We add complementary columns; existing column aliases documented inline. +-- +-- Source refs: +-- Cross_Source_Matching_Strategy sec 2.1 (house_sources columns) +-- Cross_Source_Matching_Strategy sec 2.2 (listing_sources columns) +-- Schema_Cian_SERP_Inventory sec 26.3 (external_valuations full shape) +-- Schema_Cian_SERP_Inventory sec 16.3 (houses_price_dynamics multi-dim structure) + +BEGIN; + +-- ============================================================================= +-- A. Extend house_sources +-- Add: ext_url, raw_payload, last_seen_at +-- (ext_source/ext_id/confidence/matched_method/matched_at already in 028) +-- ============================================================================= + +ALTER TABLE house_sources + ADD COLUMN IF NOT EXISTS ext_url text, -- canonical URL to the source listing/page + ADD COLUMN IF NOT EXISTS raw_payload jsonb, -- full source-side payload snapshot for diffing + ADD COLUMN IF NOT EXISTS last_seen_at timestamptz; -- last time this house was observed from this source + +-- GIN index for raw_payload JSON querying (diff, field extraction) +CREATE INDEX IF NOT EXISTS house_sources_raw_payload_gin_idx + ON house_sources USING GIN (raw_payload) + WHERE raw_payload IS NOT NULL; + +-- Recency index for stale-data detection per source +CREATE INDEX IF NOT EXISTS house_sources_last_seen_idx + ON house_sources (last_seen_at DESC) + WHERE last_seen_at IS NOT NULL; + +-- ============================================================================= +-- B. Extend listing_sources +-- Add: price_rub, area_m2, floor, rooms_count, raw_payload, last_seen_at +-- (ext_source/ext_id/confidence/matched_method/source_url/last_scraped_at already in 028) +-- Note: last_seen_at is a semantic alias for last_scraped_at — both kept for compat. +-- New code should write to last_seen_at; last_scraped_at preserved for existing queries. +-- ============================================================================= + +ALTER TABLE listing_sources + ADD COLUMN IF NOT EXISTS price_rub bigint, -- price from this source at last scrape (cross-source divergence) + ADD COLUMN IF NOT EXISTS area_m2 numeric(8,2), -- area as seen on this source + ADD COLUMN IF NOT EXISTS floor int, -- floor from this source + ADD COLUMN IF NOT EXISTS rooms_count int, -- rooms count from this source + ADD COLUMN IF NOT EXISTS raw_payload jsonb, -- full source-side payload snapshot + ADD COLUMN IF NOT EXISTS last_seen_at timestamptz; -- last time we saw this listing on source + +-- GIN index for raw_payload JSON querying +CREATE INDEX IF NOT EXISTS listing_sources_raw_payload_gin_idx + ON listing_sources USING GIN (raw_payload) + WHERE raw_payload IS NOT NULL; + +-- Recency index (complement existing listing_sources_scraped_idx on last_scraped_at) +CREATE INDEX IF NOT EXISTS listing_sources_last_seen_idx + ON listing_sources (last_seen_at DESC) + WHERE last_seen_at IS NOT NULL; + +-- ============================================================================= +-- C. Extend external_valuations +-- Add: sale_price_rub, sale_accuracy, rent_price_rub, rent_accuracy, +-- chart_change_pct, chart_change_direction, filters_hash, external_house_id +-- Existing: price_rub (primary estimate), accuracy (generic), chart, raw_payload +-- New columns provide sale/rent split and Cian-specific metadata. +-- ============================================================================= + +ALTER TABLE external_valuations + -- Sale estimate (split from generic price_rub / accuracy) + ADD COLUMN IF NOT EXISTS sale_price_rub numeric, -- sale price estimate + ADD COLUMN IF NOT EXISTS sale_accuracy numeric, -- sale accuracy 0-100 (Cian-specific) + -- Rent estimate (Cian only) + ADD COLUMN IF NOT EXISTS rent_price_rub numeric, -- rent price per month estimate + ADD COLUMN IF NOT EXISTS rent_accuracy numeric, -- rent accuracy 0-100 + -- Chart-level summary (from estimationChart) + ADD COLUMN IF NOT EXISTS chart_change_pct numeric(5,2), -- price change % since chart start + ADD COLUMN IF NOT EXISTS chart_change_direction text, -- 'increase' / 'decrease' / 'neutral' + -- Cian internal metadata + ADD COLUMN IF NOT EXISTS filters_hash text, -- sha256 of normalized filter params (dedup slight variations) + ADD COLUMN IF NOT EXISTS external_house_id bigint; -- Cian's internal houseId (filters.houseId) + +-- Index on filters_hash for dedup queries +CREATE INDEX IF NOT EXISTS ev_filters_hash_idx + ON external_valuations (filters_hash) + WHERE filters_hash IS NOT NULL; + +-- Index on external_house_id for Cian house matching lookups +CREATE INDEX IF NOT EXISTS ev_external_house_id_idx + ON external_valuations (external_house_id) + WHERE external_house_id IS NOT NULL; + +-- ============================================================================= +-- D. Extend houses_price_dynamics +-- Add: room_count text, prices_type text, period text (dimension columns) +-- Replace UNIQUE(house_id, month_date, source) with multi-dim UNIQUE. +-- +-- room_count uses Cian API enum text values: +-- 'studio'/'oneRoom'/'twoRoom'/'threeRoom'/'fourPlusRooms' / 'all' (aggregate) +-- prices_type: 'price' (total) / 'priceSqm' (per sqm) +-- period: 'halfYear' / 'year' / 'allTime' — chart time window +-- ============================================================================= + +ALTER TABLE houses_price_dynamics + ADD COLUMN IF NOT EXISTS room_count text, -- Cian room enum or 'all' for aggregate + ADD COLUMN IF NOT EXISTS prices_type text, -- 'price' / 'priceSqm' + ADD COLUMN IF NOT EXISTS period text; -- 'halfYear' / 'year' / 'allTime' + +-- Backfill existing rows so they have non-NULL dimension values. +-- Historical rows had no dimension granularity; mark as aggregate/default. +UPDATE houses_price_dynamics +SET + room_count = COALESCE(room_count, 'all'), + prices_type = COALESCE(prices_type, 'priceSqm'), + period = COALESCE(period, 'allTime') +WHERE room_count IS NULL OR prices_type IS NULL OR period IS NULL; + +-- Replace old 3-column UNIQUE with 6-column multi-dimension version. +-- Idempotent: checks pg_constraint by name before each action. +DO $$ +BEGIN + -- Drop old constraint if it still exists (default PG naming convention) + IF EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'houses_price_dynamics_house_id_month_date_source_key' + AND conrelid = 'houses_price_dynamics'::regclass + ) THEN + ALTER TABLE houses_price_dynamics + DROP CONSTRAINT houses_price_dynamics_house_id_month_date_source_key; + END IF; + + -- Add new constraint only if not already present (safe re-run) + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'houses_price_dynamics_dim_key' + AND conrelid = 'houses_price_dynamics'::regclass + ) THEN + ALTER TABLE houses_price_dynamics + ADD CONSTRAINT houses_price_dynamics_dim_key + UNIQUE (house_id, source, room_count, prices_type, period, month_date); + END IF; +END +$$; + +-- Supporting index for multi-dim read pattern (complements existing hpd_house_date_idx) +CREATE INDEX IF NOT EXISTS hpd_house_dim_idx + ON houses_price_dynamics (house_id, source, room_count, prices_type, period, month_date DESC); + +-- ============================================================================= +-- E. Extend houses — add cadastral_number for Tier-0 house matching +-- Enables: SELECT id FROM houses WHERE cadastral_number = :ck (confidence=1.0) +-- Sources: Cian offer.buildingCadastralNumber, cad_buildings.kadastr_num, DomRF KN +-- Format: '66:41:0000000:12345' (standard cadastral format) +-- ============================================================================= + +ALTER TABLE houses + ADD COLUMN IF NOT EXISTS cadastral_number text; -- building cadastral number + +-- BTree index for exact Tier-0 match lookups +CREATE INDEX IF NOT EXISTS houses_cadastral_number_idx + ON houses (cadastral_number) + WHERE cadastral_number IS NOT NULL; + +COMMIT; -- 2.45.3