feat(tradein): SQL migration 029 — extend matching/valuation/dynamics tables (PR #445 follow-up) (#448)

tradein-mvp/backend/data/sql/029_extend_matching_valuation_dynamics.sql

@@ -0,0 +1,184 @@
+-- 029_extend_matching_valuation_dynamics.sql
+-- Purpose: Extend matching/valuation/dynamics tables per PR #445 deep-code-reviewer follow-ups.
+--          Addresses 3 MEDIUM blockers:
+--            A. house_sources   — add ext_url / raw_payload / last_seen_at (Stage 8 conflict resolution)
+--            B. listing_sources — add price_rub / area_m2 / floor / rooms_count / raw_payload / last_seen_at
+--            C. external_valuations — split sale/rent + chart_change_pct/dir + filters_hash + external_house_id
+--               (Stage 7 Cian Valuation full save)
+--            D. houses_price_dynamics — add room_count / prices_type / period dimensions + replace UNIQUE
+--               (Stage 6 newbuilding multi-dim chart)
+--            E. houses — add cadastral_number for Tier-0 Cian Valuation house matching
+--
+-- Dependencies (must already exist):
+--   - house_sources       (028_matching_tables.sql)
+--   - listing_sources     (028_matching_tables.sql)
+--   - external_valuations (026_external_valuations.sql)
+--   - houses_price_dynamics (024_houses_price_dynamics.sql)
+--   - houses              (009_houses.sql)
+--
+-- Deploy order: Apply after 028.
+-- Re-run safe: all ADD COLUMN IF NOT EXISTS; DO $$ blocks guard constraint changes.
+--
+-- NOTE on column naming: house_sources uses (ext_source, ext_id) per 028 DDL.
+--                        listing_sources uses (ext_source, ext_id, source_url, last_scraped_at) per 028 DDL.
+--                        We add complementary columns; existing column aliases documented inline.
+--
+-- Source refs:
+--   Cross_Source_Matching_Strategy sec 2.1 (house_sources columns)
+--   Cross_Source_Matching_Strategy sec 2.2 (listing_sources columns)
+--   Schema_Cian_SERP_Inventory sec 26.3 (external_valuations full shape)
+--   Schema_Cian_SERP_Inventory sec 16.3 (houses_price_dynamics multi-dim structure)
+
+BEGIN;
+
+-- =============================================================================
+-- A. Extend house_sources
+--    Add: ext_url, raw_payload, last_seen_at
+--    (ext_source/ext_id/confidence/matched_method/matched_at already in 028)
+-- =============================================================================
+
+ALTER TABLE house_sources
+  ADD COLUMN IF NOT EXISTS ext_url      text,            -- canonical URL to the source listing/page
+  ADD COLUMN IF NOT EXISTS raw_payload  jsonb,           -- full source-side payload snapshot for diffing
+  ADD COLUMN IF NOT EXISTS last_seen_at timestamptz;     -- last time this house was observed from this source
+
+-- GIN index for raw_payload JSON querying (diff, field extraction)
+CREATE INDEX IF NOT EXISTS house_sources_raw_payload_gin_idx
+  ON house_sources USING GIN (raw_payload)
+  WHERE raw_payload IS NOT NULL;
+
+-- Recency index for stale-data detection per source
+CREATE INDEX IF NOT EXISTS house_sources_last_seen_idx
+  ON house_sources (last_seen_at DESC)
+  WHERE last_seen_at IS NOT NULL;
+
+-- =============================================================================
+-- B. Extend listing_sources
+--    Add: price_rub, area_m2, floor, rooms_count, raw_payload, last_seen_at
+--    (ext_source/ext_id/confidence/matched_method/source_url/last_scraped_at already in 028)
+--    Note: last_seen_at is a semantic alias for last_scraped_at — both kept for compat.
+--          New code should write to last_seen_at; last_scraped_at preserved for existing queries.
+-- =============================================================================
+
+ALTER TABLE listing_sources
+  ADD COLUMN IF NOT EXISTS price_rub    bigint,          -- price from this source at last scrape (cross-source divergence)
+  ADD COLUMN IF NOT EXISTS area_m2      numeric(8,2),    -- area as seen on this source
+  ADD COLUMN IF NOT EXISTS floor        int,             -- floor from this source
+  ADD COLUMN IF NOT EXISTS rooms_count  int,             -- rooms count from this source
+  ADD COLUMN IF NOT EXISTS raw_payload  jsonb,           -- full source-side payload snapshot
+  ADD COLUMN IF NOT EXISTS last_seen_at timestamptz;     -- last time we saw this listing on source
+
+-- GIN index for raw_payload JSON querying
+CREATE INDEX IF NOT EXISTS listing_sources_raw_payload_gin_idx
+  ON listing_sources USING GIN (raw_payload)
+  WHERE raw_payload IS NOT NULL;
+
+-- Recency index (complement existing listing_sources_scraped_idx on last_scraped_at)
+CREATE INDEX IF NOT EXISTS listing_sources_last_seen_idx
+  ON listing_sources (last_seen_at DESC)
+  WHERE last_seen_at IS NOT NULL;
+
+-- =============================================================================
+-- C. Extend external_valuations
+--    Add: sale_price_rub, sale_accuracy, rent_price_rub, rent_accuracy,
+--         chart_change_pct, chart_change_direction, filters_hash, external_house_id
+--    Existing: price_rub (primary estimate), accuracy (generic), chart, raw_payload
+--    New columns provide sale/rent split and Cian-specific metadata.
+-- =============================================================================
+
+ALTER TABLE external_valuations
+  -- Sale estimate (split from generic price_rub / accuracy)
+  ADD COLUMN IF NOT EXISTS sale_price_rub         numeric,      -- sale price estimate
+  ADD COLUMN IF NOT EXISTS sale_accuracy          numeric,      -- sale accuracy 0-100 (Cian-specific)
+  -- Rent estimate (Cian only)
+  ADD COLUMN IF NOT EXISTS rent_price_rub         numeric,      -- rent price per month estimate
+  ADD COLUMN IF NOT EXISTS rent_accuracy          numeric,      -- rent accuracy 0-100
+  -- Chart-level summary (from estimationChart)
+  ADD COLUMN IF NOT EXISTS chart_change_pct       numeric(5,2), -- price change % since chart start
+  ADD COLUMN IF NOT EXISTS chart_change_direction text,         -- 'increase' / 'decrease' / 'neutral'
+  -- Cian internal metadata
+  ADD COLUMN IF NOT EXISTS filters_hash           text,         -- sha256 of normalized filter params (dedup slight variations)
+  ADD COLUMN IF NOT EXISTS external_house_id      bigint;       -- Cian's internal houseId (filters.houseId)
+
+-- Index on filters_hash for dedup queries
+CREATE INDEX IF NOT EXISTS ev_filters_hash_idx
+  ON external_valuations (filters_hash)
+  WHERE filters_hash IS NOT NULL;
+
+-- Index on external_house_id for Cian house matching lookups
+CREATE INDEX IF NOT EXISTS ev_external_house_id_idx
+  ON external_valuations (external_house_id)
+  WHERE external_house_id IS NOT NULL;
+
+-- =============================================================================
+-- D. Extend houses_price_dynamics
+--    Add: room_count text, prices_type text, period text (dimension columns)
+--    Replace UNIQUE(house_id, month_date, source) with multi-dim UNIQUE.
+--
+--    room_count uses Cian API enum text values:
+--      'studio'/'oneRoom'/'twoRoom'/'threeRoom'/'fourPlusRooms' / 'all' (aggregate)
+--    prices_type: 'price' (total) / 'priceSqm' (per sqm)
+--    period: 'halfYear' / 'year' / 'allTime' — chart time window
+-- =============================================================================
+
+ALTER TABLE houses_price_dynamics
+  ADD COLUMN IF NOT EXISTS room_count   text,   -- Cian room enum or 'all' for aggregate
+  ADD COLUMN IF NOT EXISTS prices_type  text,   -- 'price' / 'priceSqm'
+  ADD COLUMN IF NOT EXISTS period       text;   -- 'halfYear' / 'year' / 'allTime'
+
+-- Backfill existing rows so they have non-NULL dimension values.
+-- Historical rows had no dimension granularity; mark as aggregate/default.
+UPDATE houses_price_dynamics
+SET
+  room_count   = COALESCE(room_count,  'all'),
+  prices_type  = COALESCE(prices_type, 'priceSqm'),
+  period       = COALESCE(period,      'allTime')
+WHERE room_count IS NULL OR prices_type IS NULL OR period IS NULL;
+
+-- Replace old 3-column UNIQUE with 6-column multi-dimension version.
+-- Idempotent: checks pg_constraint by name before each action.
+DO $$
+BEGIN
+    -- Drop old constraint if it still exists (default PG naming convention)
+    IF EXISTS (
+        SELECT 1 FROM pg_constraint
+        WHERE conname = 'houses_price_dynamics_house_id_month_date_source_key'
+          AND conrelid = 'houses_price_dynamics'::regclass
+    ) THEN
+        ALTER TABLE houses_price_dynamics
+            DROP CONSTRAINT houses_price_dynamics_house_id_month_date_source_key;
+    END IF;
+
+    -- Add new constraint only if not already present (safe re-run)
+    IF NOT EXISTS (
+        SELECT 1 FROM pg_constraint
+        WHERE conname = 'houses_price_dynamics_dim_key'
+          AND conrelid = 'houses_price_dynamics'::regclass
+    ) THEN
+        ALTER TABLE houses_price_dynamics
+            ADD CONSTRAINT houses_price_dynamics_dim_key
+            UNIQUE (house_id, source, room_count, prices_type, period, month_date);
+    END IF;
+END
+$$;
+
+-- Supporting index for multi-dim read pattern (complements existing hpd_house_date_idx)
+CREATE INDEX IF NOT EXISTS hpd_house_dim_idx
+  ON houses_price_dynamics (house_id, source, room_count, prices_type, period, month_date DESC);
+
+-- =============================================================================
+-- E. Extend houses — add cadastral_number for Tier-0 house matching
+--    Enables: SELECT id FROM houses WHERE cadastral_number = :ck  (confidence=1.0)
+--    Sources: Cian offer.buildingCadastralNumber, cad_buildings.kadastr_num, DomRF KN
+--    Format: '66:41:0000000:12345' (standard cadastral format)
+-- =============================================================================
+
+ALTER TABLE houses
+  ADD COLUMN IF NOT EXISTS cadastral_number text;  -- building cadastral number
+
+-- BTree index for exact Tier-0 match lookups
+CREATE INDEX IF NOT EXISTS houses_cadastral_number_idx
+  ON houses (cadastral_number)
+  WHERE cadastral_number IS NOT NULL;
+
+COMMIT;