-- 029_extend_matching_valuation_dynamics.sql
-- Purpose: Extend matching/valuation/dynamics tables per PR #445 deep-code-reviewer follow-ups.
--          Addresses 3 MEDIUM blockers:
--            A. house_sources   — add ext_url / raw_payload / last_seen_at (Stage 8 conflict resolution)
--            B. listing_sources — add price_rub / area_m2 / floor / rooms_count / raw_payload / last_seen_at
--            C. external_valuations — split sale/rent + chart_change_pct/dir + filters_hash + external_house_id
--               (Stage 7 Cian Valuation full save)
--            D. houses_price_dynamics — add room_count / prices_type / period dimensions + replace UNIQUE
--               (Stage 6 newbuilding multi-dim chart)
--            E. houses — add cadastral_number for Tier-0 Cian Valuation house matching
--
-- Dependencies (must already exist):
--   - house_sources       (028_matching_tables.sql)
--   - listing_sources     (028_matching_tables.sql)
--   - external_valuations (026_external_valuations.sql)
--   - houses_price_dynamics (024_houses_price_dynamics.sql)
--   - houses              (009_houses.sql)
--
-- Deploy order: Apply after 028.
-- Re-run safe: all ADD COLUMN IF NOT EXISTS; DO $$ blocks guard constraint changes.
--
-- NOTE on column naming: house_sources uses (ext_source, ext_id) per 028 DDL.
--                        listing_sources uses (ext_source, ext_id, source_url, last_scraped_at) per 028 DDL.
--                        We add complementary columns; existing column aliases documented inline.
--
-- Source refs:
--   Cross_Source_Matching_Strategy sec 2.1 (house_sources columns)
--   Cross_Source_Matching_Strategy sec 2.2 (listing_sources columns)
--   Schema_Cian_SERP_Inventory sec 26.3 (external_valuations full shape)
--   Schema_Cian_SERP_Inventory sec 16.3 (houses_price_dynamics multi-dim structure)

BEGIN;

-- =============================================================================
-- A. Extend house_sources
--    Add: ext_url, raw_payload, last_seen_at
--    (ext_source/ext_id/confidence/matched_method/matched_at already in 028)
-- =============================================================================

ALTER TABLE house_sources
  ADD COLUMN IF NOT EXISTS ext_url      text,            -- canonical URL to the source listing/page
  ADD COLUMN IF NOT EXISTS raw_payload  jsonb,           -- full source-side payload snapshot for diffing
  ADD COLUMN IF NOT EXISTS last_seen_at timestamptz;     -- last time this house was observed from this source

-- GIN index for raw_payload JSON querying (diff, field extraction)
CREATE INDEX IF NOT EXISTS house_sources_raw_payload_gin_idx
  ON house_sources USING GIN (raw_payload)
  WHERE raw_payload IS NOT NULL;

-- Recency index for stale-data detection per source
CREATE INDEX IF NOT EXISTS house_sources_last_seen_idx
  ON house_sources (last_seen_at DESC)
  WHERE last_seen_at IS NOT NULL;

-- =============================================================================
-- B. Extend listing_sources
--    Add: price_rub, area_m2, floor, rooms_count, raw_payload, last_seen_at
--    (ext_source/ext_id/confidence/matched_method/source_url/last_scraped_at already in 028)
--    Note: last_seen_at is a semantic alias for last_scraped_at — both kept for compat.
--          New code should write to last_seen_at; last_scraped_at preserved for existing queries.
-- =============================================================================

ALTER TABLE listing_sources
  ADD COLUMN IF NOT EXISTS price_rub    bigint,          -- price from this source at last scrape (cross-source divergence)
  ADD COLUMN IF NOT EXISTS area_m2      numeric(8,2),    -- area as seen on this source
  ADD COLUMN IF NOT EXISTS floor        int,             -- floor from this source
  ADD COLUMN IF NOT EXISTS rooms_count  int,             -- rooms count from this source
  ADD COLUMN IF NOT EXISTS raw_payload  jsonb,           -- full source-side payload snapshot
  ADD COLUMN IF NOT EXISTS last_seen_at timestamptz;     -- last time we saw this listing on source

-- GIN index for raw_payload JSON querying
CREATE INDEX IF NOT EXISTS listing_sources_raw_payload_gin_idx
  ON listing_sources USING GIN (raw_payload)
  WHERE raw_payload IS NOT NULL;

-- Recency index (complement existing listing_sources_scraped_idx on last_scraped_at)
CREATE INDEX IF NOT EXISTS listing_sources_last_seen_idx
  ON listing_sources (last_seen_at DESC)
  WHERE last_seen_at IS NOT NULL;

-- =============================================================================
-- C. Extend external_valuations
--    Add: sale_price_rub, sale_accuracy, rent_price_rub, rent_accuracy,
--         chart_change_pct, chart_change_direction, filters_hash, external_house_id
--    Existing: price_rub (primary estimate), accuracy (generic), chart, raw_payload
--    New columns provide sale/rent split and Cian-specific metadata.
-- =============================================================================

ALTER TABLE external_valuations
  -- Sale estimate (split from generic price_rub / accuracy)
  ADD COLUMN IF NOT EXISTS sale_price_rub         numeric,      -- sale price estimate
  ADD COLUMN IF NOT EXISTS sale_accuracy          numeric,      -- sale accuracy 0-100 (Cian-specific)
  -- Rent estimate (Cian only)
  ADD COLUMN IF NOT EXISTS rent_price_rub         numeric,      -- rent price per month estimate
  ADD COLUMN IF NOT EXISTS rent_accuracy          numeric,      -- rent accuracy 0-100
  -- Chart-level summary (from estimationChart)
  ADD COLUMN IF NOT EXISTS chart_change_pct       numeric(5,2), -- price change % since chart start
  ADD COLUMN IF NOT EXISTS chart_change_direction text,         -- 'increase' / 'decrease' / 'neutral'
  -- Cian internal metadata
  ADD COLUMN IF NOT EXISTS filters_hash           text,         -- sha256 of normalized filter params (dedup slight variations)
  ADD COLUMN IF NOT EXISTS external_house_id      bigint;       -- Cian's internal houseId (filters.houseId)

-- Index on filters_hash for dedup queries
CREATE INDEX IF NOT EXISTS ev_filters_hash_idx
  ON external_valuations (filters_hash)
  WHERE filters_hash IS NOT NULL;

-- Index on external_house_id for Cian house matching lookups
CREATE INDEX IF NOT EXISTS ev_external_house_id_idx
  ON external_valuations (external_house_id)
  WHERE external_house_id IS NOT NULL;

-- =============================================================================
-- D. Extend houses_price_dynamics
--    Add: room_count text, prices_type text, period text (dimension columns)
--    Replace UNIQUE(house_id, month_date, source) with multi-dim UNIQUE.
--
--    room_count uses Cian API enum text values:
--      'studio'/'oneRoom'/'twoRoom'/'threeRoom'/'fourPlusRooms' / 'all' (aggregate)
--    prices_type: 'price' (total) / 'priceSqm' (per sqm)
--    period: 'halfYear' / 'year' / 'allTime' — chart time window
-- =============================================================================

ALTER TABLE houses_price_dynamics
  ADD COLUMN IF NOT EXISTS room_count   text,   -- Cian room enum or 'all' for aggregate
  ADD COLUMN IF NOT EXISTS prices_type  text,   -- 'price' / 'priceSqm'
  ADD COLUMN IF NOT EXISTS period       text;   -- 'halfYear' / 'year' / 'allTime'

-- Backfill existing rows so they have non-NULL dimension values.
-- Historical rows had no dimension granularity; mark as aggregate/default.
UPDATE houses_price_dynamics
SET
  room_count   = COALESCE(room_count,  'all'),
  prices_type  = COALESCE(prices_type, 'priceSqm'),
  period       = COALESCE(period,      'allTime')
WHERE room_count IS NULL OR prices_type IS NULL OR period IS NULL;

-- Replace old 3-column UNIQUE with 6-column multi-dimension version.
-- Idempotent: checks pg_constraint by name before each action.
DO $$
BEGIN
    -- Drop old constraint if it still exists (default PG naming convention)
    IF EXISTS (
        SELECT 1 FROM pg_constraint
        WHERE conname = 'houses_price_dynamics_house_id_month_date_source_key'
          AND conrelid = 'houses_price_dynamics'::regclass
    ) THEN
        ALTER TABLE houses_price_dynamics
            DROP CONSTRAINT houses_price_dynamics_house_id_month_date_source_key;
    END IF;

    -- Add new constraint only if not already present (safe re-run)
    IF NOT EXISTS (
        SELECT 1 FROM pg_constraint
        WHERE conname = 'houses_price_dynamics_dim_key'
          AND conrelid = 'houses_price_dynamics'::regclass
    ) THEN
        ALTER TABLE houses_price_dynamics
            ADD CONSTRAINT houses_price_dynamics_dim_key
            UNIQUE (house_id, source, room_count, prices_type, period, month_date);
    END IF;
END
$$;

-- Supporting index for multi-dim read pattern (complements existing hpd_house_date_idx)
CREATE INDEX IF NOT EXISTS hpd_house_dim_idx
  ON houses_price_dynamics (house_id, source, room_count, prices_type, period, month_date DESC);

-- =============================================================================
-- E. Extend houses — add cadastral_number for Tier-0 house matching
--    Enables: SELECT id FROM houses WHERE cadastral_number = :ck  (confidence=1.0)
--    Sources: Cian offer.buildingCadastralNumber, cad_buildings.kadastr_num, DomRF KN
--    Format: '66:41:0000000:12345' (standard cadastral format)
-- =============================================================================

ALTER TABLE houses
  ADD COLUMN IF NOT EXISTS cadastral_number text;  -- building cadastral number

-- BTree index for exact Tier-0 match lookups
CREATE INDEX IF NOT EXISTS houses_cadastral_number_idx
  ON houses (cadastral_number)
  WHERE cadastral_number IS NOT NULL;

COMMIT;