feat(tradein): SQL 040-046 — matching schema delta (Phase 1.1) (#464)

tradein-mvp/backend/data/sql/040_houses_extend.sql

@@ -0,0 +1,26 @@
+-- 040_houses_extend.sql
+-- Purpose: Add complex_id FK placeholder + per-source enrichment timestamps to houses.
+--          Most planned columns (address_fingerprint, cadastral_number, commission_year/month,
+--          corpus_count, total_area_ha, has_panorama, yandex_jk_id/slug, yandex_validated_at,
+--          cian_internal_house_id) already added in earlier migrations (028/029/020/031).
+-- Dependencies: 009_houses.sql (houses table)
+-- Apply after: 032_yandex_history.sql
+
+BEGIN;
+
+ALTER TABLE houses
+  ADD COLUMN IF NOT EXISTS complex_id          bigint,
+  ADD COLUMN IF NOT EXISTS avito_validated_at  timestamptz,
+  ADD COLUMN IF NOT EXISTS cian_validated_at   timestamptz;
+
+CREATE INDEX IF NOT EXISTS houses_complex_id_idx
+  ON houses (complex_id) WHERE complex_id IS NOT NULL;
+
+COMMENT ON COLUMN houses.complex_id IS
+  'FK placeholder for future complexes (ЖК-level) table. NULL until ЖК aggregation introduced.';
+COMMENT ON COLUMN houses.avito_validated_at IS
+  'Timestamp of last enrichment from Avito Houses Catalog (mirror of yandex_validated_at semantics).';
+COMMENT ON COLUMN houses.cian_validated_at IS
+  'Timestamp of last enrichment from Cian newbuilding/BTI (mirror of yandex_validated_at semantics).';
+
+COMMIT;

tradein-mvp/backend/data/sql/041_house_sources_noop.sql

@@ -0,0 +1,16 @@
+-- 041_house_sources_noop.sql
+-- Purpose: Document that house_sources already exists (028 + 029) and satisfies
+--          Cross_Source_Matching_Strategy sec 2.1. Column-name aliases noted in COMMENTs.
+-- Dependencies: 028_matching_tables.sql, 029_extend_matching_valuation_dynamics.sql
+-- No DDL changes — idempotent COMMENT-only file.
+
+BEGIN;
+
+COMMENT ON COLUMN house_sources.ext_source IS
+  'Source identifier (alias of "source" in Master Plan sec 2.1). Values: avito/cian/yandex/n1.';
+COMMENT ON COLUMN house_sources.ext_id IS
+  'External house id from source (alias of "ext_house_id" in Master Plan).';
+COMMENT ON COLUMN house_sources.matched_method IS
+  'Match method (alias of "match_method" in Master Plan). Values: cadastr_exact/address_geo_year/geo_only/new/manual.';
+
+COMMIT;

tradein-mvp/backend/data/sql/042_listing_sources_price_divergence_idx.sql

@@ -0,0 +1,12 @@
+-- 042_listing_sources_price_divergence_idx.sql
+-- Purpose: Add price-divergence index (Master Plan sec 5.1 idx #9).
+--          listing_sources columns/canonical/merged_into already provisioned in 028 + 029.
+-- Dependencies: 028_matching_tables.sql, 029_extend_matching_valuation_dynamics.sql
+
+BEGIN;
+
+CREATE INDEX IF NOT EXISTS listing_sources_price_divergence_idx
+  ON listing_sources (listing_id, price_rub)
+  WHERE price_rub IS NOT NULL;
+
+COMMIT;

tradein-mvp/backend/data/sql/043_house_reviews_extend.sql

@@ -0,0 +1,14 @@
+-- 043_house_reviews_extend.sql
+-- Purpose: Add 'likes' column to house_reviews per Cross_Source_Matching sec 15.
+--          Table itself exists in 014 with richer split (text_main/pros/cons vs single "text").
+-- Dependencies: 014_house_reviews.sql
+
+BEGIN;
+
+ALTER TABLE house_reviews
+  ADD COLUMN IF NOT EXISTS likes int;
+
+COMMENT ON COLUMN house_reviews.likes IS
+  'Likes/helpful count from source (Cian / Yandex). NULL for Avito (not exposed).';
+
+COMMIT;

tradein-mvp/backend/data/sql/044_external_valuations_link.sql

@@ -0,0 +1,37 @@
+-- 044_external_valuations_link.sql
+-- Purpose: Link external_valuations to canonical entities (house/listing) + add
+--          sale band columns (low/high) per Cross_Source_Matching sec 13.1.
+--          Table base in 026 + 029. Existing UNIQUE (source, cache_key) preserved.
+-- Dependencies: 026_external_valuations.sql, 029_extend_matching_valuation_dynamics.sql,
+--               009_houses.sql, 002_core_tables.sql (listings)
+
+BEGIN;
+
+ALTER TABLE external_valuations
+  ADD COLUMN IF NOT EXISTS house_id   bigint REFERENCES houses(id)   ON DELETE CASCADE,
+  ADD COLUMN IF NOT EXISTS listing_id bigint REFERENCES listings(id) ON DELETE CASCADE,
+  ADD COLUMN IF NOT EXISTS low_price  bigint,
+  ADD COLUMN IF NOT EXISTS high_price bigint;
+
+CREATE INDEX IF NOT EXISTS external_valuations_house_idx
+  ON external_valuations (house_id)
+  WHERE house_id IS NOT NULL;
+
+CREATE INDEX IF NOT EXISTS external_valuations_listing_idx
+  ON external_valuations (listing_id)
+  WHERE listing_id IS NOT NULL;
+
+CREATE UNIQUE INDEX IF NOT EXISTS external_valuations_canonical_uniq_idx
+  ON external_valuations (source, house_id, listing_id)
+  WHERE house_id IS NOT NULL AND listing_id IS NOT NULL;
+
+COMMENT ON COLUMN external_valuations.house_id IS
+  'FK to canonical house (post-matching). NULL for pre-matching cache rows keyed by cache_key.';
+COMMENT ON COLUMN external_valuations.listing_id IS
+  'FK to canonical listing (post-matching). NULL for house-level estimates.';
+COMMENT ON COLUMN external_valuations.low_price IS
+  'Lower bound of sale band (Cian sometimes returns; NULL for Avito IMV).';
+COMMENT ON COLUMN external_valuations.high_price IS
+  'Upper bound of sale band.';
+
+COMMIT;

tradein-mvp/backend/data/sql/045_house_placement_history_extend.sql

@@ -0,0 +1,27 @@
+-- 045_house_placement_history_extend.sql
+-- Purpose: Add source_confidence + notes + per-house 3-col UNIQUE per Cross_Source sec 14.1.
+--          Table base in 017; source/ext_item_id already present.
+--          Note: plan calls the external-id column "ext_listing_id", but existing
+--          schema uses "ext_item_id" — semantic alias documented in COMMENT.
+-- Dependencies: 017_house_placement_history.sql, 032_yandex_history.sql
+
+BEGIN;
+
+ALTER TABLE house_placement_history
+  ADD COLUMN IF NOT EXISTS source_confidence real DEFAULT 1.0,
+  ADD COLUMN IF NOT EXISTS notes             text;
+
+CREATE UNIQUE INDEX IF NOT EXISTS hph_house_source_extid_uniq_idx
+  ON house_placement_history (house_id, source, ext_item_id)
+  WHERE house_id IS NOT NULL;
+
+COMMENT ON COLUMN house_placement_history.source_confidence IS
+  'Confidence (0-1) that this history row belongs to the linked house. '
+  '1.0 for direct Avito Houses Catalog / Cian offer; lower for Yandex Valuation synthetic IDs.';
+COMMENT ON COLUMN house_placement_history.notes IS
+  'Free-form notes — used by dedup logic (e.g. ''dup_of:cian_offer:123''). '
+  'See Cross_Source_Matching_Strategy sec 14.3.';
+COMMENT ON COLUMN house_placement_history.ext_item_id IS
+  'External listing id (alias of "ext_listing_id" in Master Plan sec 14.1).';
+
+COMMIT;

tradein-mvp/backend/data/sql/046_views.sql

@@ -0,0 +1,100 @@
+-- 046_views.sql
+-- Purpose: Analytical / monitoring views for Phase 1.1.
+--   - v_price_divergence    (Cross_Source_Matching sec 2.2 + 4.6) — cross-source price disagreement
+--   - v_cross_source_health (Master Plan sec 8.4)                 — listing-level spread metrics
+--   - v_data_quality        (Master Plan sec 8.1)                 — KPI dashboard rows
+-- Dependencies: house_sources, listing_sources, houses, listings, house_placement_history.
+-- All CREATE OR REPLACE — idempotent.
+
+BEGIN;
+
+-- ─────────────────────────────────────────────────────────────────
+-- v_price_divergence — Cross_Source sec 2.2: 5%+ spread between sources for same canonical listing
+-- ─────────────────────────────────────────────────────────────────
+CREATE OR REPLACE VIEW v_price_divergence AS
+SELECT
+    listing_id,
+    ARRAY_AGG(DISTINCT ext_source) AS sources,
+    MAX(price_rub) - MIN(price_rub) AS price_spread,
+    (MAX(price_rub) - MIN(price_rub))::float / NULLIF(MIN(price_rub), 0) * 100 AS spread_pct,
+    COUNT(*) AS source_count
+FROM listing_sources
+WHERE price_rub IS NOT NULL
+GROUP BY listing_id
+HAVING (MAX(price_rub) - MIN(price_rub))::float / NULLIF(MIN(price_rub), 0) > 0.05;
+
+COMMENT ON VIEW v_price_divergence IS
+  'Listings with cross-source price spread > 5%. Used by quality monitor + manual review queue.';
+
+-- ─────────────────────────────────────────────────────────────────
+-- v_cross_source_health — Master Plan sec 8.4: listing-level spread breakdown
+-- ─────────────────────────────────────────────────────────────────
+CREATE OR REPLACE VIEW v_cross_source_health AS
+SELECT
+    listing_id,
+    array_agg(ext_source ORDER BY price_rub)     AS sources,
+    array_agg(price_rub  ORDER BY price_rub)     AS prices,
+    max(price_rub) - min(price_rub)              AS spread,
+    (max(price_rub) - min(price_rub))::float
+        / NULLIF(min(price_rub), 0) * 100        AS spread_pct,
+    count(*)                                     AS source_count
+FROM listing_sources
+WHERE price_rub IS NOT NULL
+GROUP BY listing_id
+HAVING count(*) >= 2;
+
+COMMENT ON VIEW v_cross_source_health IS
+  'Per-listing cross-source price aggregation. Alert if spread_pct > 15% on > 100 listings.';
+
+-- ─────────────────────────────────────────────────────────────────
+-- v_data_quality — Master Plan sec 8.1: single-row KPI snapshot
+-- ─────────────────────────────────────────────────────────────────
+CREATE OR REPLACE VIEW v_data_quality AS
+WITH active_listings AS (
+    SELECT * FROM listings WHERE is_active = true
+)
+SELECT
+    (SELECT count(*) FROM houses)                                                  AS houses_total,
+    (SELECT count(*) FROM houses h
+       WHERE EXISTS (SELECT 1 FROM house_sources hs WHERE hs.house_id = h.id))      AS houses_with_source,
+    (SELECT count(*) FROM houses h
+       WHERE EXISTS (SELECT 1 FROM house_sources hs
+                      WHERE hs.house_id = h.id AND hs.ext_source = 'avito'))        AS houses_with_avito,
+    (SELECT count(*) FROM houses h
+       WHERE EXISTS (SELECT 1 FROM house_sources hs
+                      WHERE hs.house_id = h.id AND hs.ext_source LIKE 'cian%'))     AS houses_with_cian,
+    (SELECT count(*) FROM houses h
+       WHERE EXISTS (SELECT 1 FROM house_sources hs
+                      WHERE hs.house_id = h.id AND hs.ext_source = 'yandex'))       AS houses_with_yandex,
+    (SELECT count(*) FROM (
+        SELECT house_id FROM house_sources GROUP BY house_id HAVING count(*) >= 2
+    ) sub)                                                                          AS houses_2plus_sources,
+    (SELECT count(*) FROM (
+        SELECT house_id FROM house_sources GROUP BY house_id HAVING count(*) >= 3
+    ) sub)                                                                          AS houses_3plus_sources,
+    (SELECT count(*) FROM active_listings)                                         AS listings_active,
+    (SELECT count(*) FROM (
+        SELECT listing_id FROM listing_sources
+         WHERE listing_id IN (SELECT id FROM active_listings)
+         GROUP BY listing_id HAVING count(*) >= 2
+    ) sub)                                                                          AS listings_dedup_2sources,
+    (SELECT count(*) FROM active_listings WHERE lat IS NOT NULL) * 100.0
+        / NULLIF((SELECT count(*) FROM active_listings), 0)                         AS pct_geocoded,
+    (SELECT count(*) FROM active_listings WHERE kadastr_num IS NOT NULL) * 100.0
+        / NULLIF((SELECT count(*) FROM active_listings), 0)                         AS pct_cadastr,
+    (SELECT count(*) FROM active_listings WHERE description IS NOT NULL) * 100.0
+        / NULLIF((SELECT count(*) FROM active_listings), 0)                         AS pct_description,
+    (SELECT count(*) FROM active_listings l
+       JOIN houses h ON h.id = l.house_id_fk
+       WHERE h.year_built IS NOT NULL) * 100.0
+        / NULLIF((SELECT count(*) FROM active_listings), 0)                         AS pct_year_built,
+    NOW() - (SELECT max(scraped_at) FROM listings WHERE source = 'avito')           AS avito_last_scrape_ago,
+    NOW() - (SELECT max(scraped_at) FROM listings WHERE source = 'cian')            AS cian_last_scrape_ago,
+    NOW() - (SELECT max(scraped_at) FROM listings WHERE source = 'yandex')          AS yandex_last_scrape_ago,
+    (SELECT count(*) FROM v_price_divergence)                                       AS price_disagreements_count,
+    (SELECT count(*) FROM listings WHERE is_outlier = true)                         AS outliers_flagged;
+
+COMMENT ON VIEW v_data_quality IS
+  'KPI snapshot. Refreshed on-demand by /api/v1/admin/data-quality endpoint (Master Plan sec 8.1).';
+
+COMMIT;