From d019b5327ec1871b407a88422fc55cc97ddcd6bf Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 16:22:58 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20SQL=20040-046=20=E2=80=94=20ma?= =?UTF-8?q?tching=20schema=20delta=20(Phase=201.1)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Multi-Source Integration Phase 1.1 thin-delta migrations. Gap audit showed 80%+ of planned schema already in 014/017/020/026/028/029/030/031 — this PR adds only the missing pieces: - 040 houses_extend: complex_id + avito_validated_at + cian_validated_at - 041 house_sources_noop: COMMENT aliases for ext_source/ext_id/matched_method - 042 listing_sources_price_divergence_idx: partial index for /search - 043 house_reviews_extend: likes column - 044 external_valuations_link: house_id/listing_id FKs + low/high price band - 045 house_placement_history_extend: source_confidence + notes + 3-col UNIQUE - 046 views: v_price_divergence + v_cross_source_health + v_data_quality All BEGIN/COMMIT, IF NOT EXISTS, FK ON DELETE CASCADE. Idempotent — safe re-apply on prod. Refs Master Plan sec 1, 4-8 + Cross_Source_Matching sec 2-4. --- .../backend/data/sql/040_houses_extend.sql | 26 +++++ .../data/sql/041_house_sources_noop.sql | 16 +++ ...2_listing_sources_price_divergence_idx.sql | 12 +++ .../data/sql/043_house_reviews_extend.sql | 14 +++ .../data/sql/044_external_valuations_link.sql | 37 +++++++ .../045_house_placement_history_extend.sql | 27 +++++ tradein-mvp/backend/data/sql/046_views.sql | 100 ++++++++++++++++++ 7 files changed, 232 insertions(+) create mode 100644 tradein-mvp/backend/data/sql/040_houses_extend.sql create mode 100644 tradein-mvp/backend/data/sql/041_house_sources_noop.sql create mode 100644 tradein-mvp/backend/data/sql/042_listing_sources_price_divergence_idx.sql create mode 100644 tradein-mvp/backend/data/sql/043_house_reviews_extend.sql create mode 100644 tradein-mvp/backend/data/sql/044_external_valuations_link.sql create mode 100644 tradein-mvp/backend/data/sql/045_house_placement_history_extend.sql create mode 100644 tradein-mvp/backend/data/sql/046_views.sql diff --git a/tradein-mvp/backend/data/sql/040_houses_extend.sql b/tradein-mvp/backend/data/sql/040_houses_extend.sql new file mode 100644 index 00000000..0086766a --- /dev/null +++ b/tradein-mvp/backend/data/sql/040_houses_extend.sql @@ -0,0 +1,26 @@ +-- 040_houses_extend.sql +-- Purpose: Add complex_id FK placeholder + per-source enrichment timestamps to houses. +-- Most planned columns (address_fingerprint, cadastral_number, commission_year/month, +-- corpus_count, total_area_ha, has_panorama, yandex_jk_id/slug, yandex_validated_at, +-- cian_internal_house_id) already added in earlier migrations (028/029/020/031). +-- Dependencies: 009_houses.sql (houses table) +-- Apply after: 032_yandex_history.sql + +BEGIN; + +ALTER TABLE houses + ADD COLUMN IF NOT EXISTS complex_id bigint, + ADD COLUMN IF NOT EXISTS avito_validated_at timestamptz, + ADD COLUMN IF NOT EXISTS cian_validated_at timestamptz; + +CREATE INDEX IF NOT EXISTS houses_complex_id_idx + ON houses (complex_id) WHERE complex_id IS NOT NULL; + +COMMENT ON COLUMN houses.complex_id IS + 'FK placeholder for future complexes (ЖК-level) table. NULL until ЖК aggregation introduced.'; +COMMENT ON COLUMN houses.avito_validated_at IS + 'Timestamp of last enrichment from Avito Houses Catalog (mirror of yandex_validated_at semantics).'; +COMMENT ON COLUMN houses.cian_validated_at IS + 'Timestamp of last enrichment from Cian newbuilding/BTI (mirror of yandex_validated_at semantics).'; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/041_house_sources_noop.sql b/tradein-mvp/backend/data/sql/041_house_sources_noop.sql new file mode 100644 index 00000000..39bb0144 --- /dev/null +++ b/tradein-mvp/backend/data/sql/041_house_sources_noop.sql @@ -0,0 +1,16 @@ +-- 041_house_sources_noop.sql +-- Purpose: Document that house_sources already exists (028 + 029) and satisfies +-- Cross_Source_Matching_Strategy sec 2.1. Column-name aliases noted in COMMENTs. +-- Dependencies: 028_matching_tables.sql, 029_extend_matching_valuation_dynamics.sql +-- No DDL changes — idempotent COMMENT-only file. + +BEGIN; + +COMMENT ON COLUMN house_sources.ext_source IS + 'Source identifier (alias of "source" in Master Plan sec 2.1). Values: avito/cian/yandex/n1.'; +COMMENT ON COLUMN house_sources.ext_id IS + 'External house id from source (alias of "ext_house_id" in Master Plan).'; +COMMENT ON COLUMN house_sources.matched_method IS + 'Match method (alias of "match_method" in Master Plan). Values: cadastr_exact/address_geo_year/geo_only/new/manual.'; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/042_listing_sources_price_divergence_idx.sql b/tradein-mvp/backend/data/sql/042_listing_sources_price_divergence_idx.sql new file mode 100644 index 00000000..aa63d286 --- /dev/null +++ b/tradein-mvp/backend/data/sql/042_listing_sources_price_divergence_idx.sql @@ -0,0 +1,12 @@ +-- 042_listing_sources_price_divergence_idx.sql +-- Purpose: Add price-divergence index (Master Plan sec 5.1 idx #9). +-- listing_sources columns/canonical/merged_into already provisioned in 028 + 029. +-- Dependencies: 028_matching_tables.sql, 029_extend_matching_valuation_dynamics.sql + +BEGIN; + +CREATE INDEX IF NOT EXISTS listing_sources_price_divergence_idx + ON listing_sources (listing_id, price_rub) + WHERE price_rub IS NOT NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/043_house_reviews_extend.sql b/tradein-mvp/backend/data/sql/043_house_reviews_extend.sql new file mode 100644 index 00000000..ab6bbb44 --- /dev/null +++ b/tradein-mvp/backend/data/sql/043_house_reviews_extend.sql @@ -0,0 +1,14 @@ +-- 043_house_reviews_extend.sql +-- Purpose: Add 'likes' column to house_reviews per Cross_Source_Matching sec 15. +-- Table itself exists in 014 with richer split (text_main/pros/cons vs single "text"). +-- Dependencies: 014_house_reviews.sql + +BEGIN; + +ALTER TABLE house_reviews + ADD COLUMN IF NOT EXISTS likes int; + +COMMENT ON COLUMN house_reviews.likes IS + 'Likes/helpful count from source (Cian / Yandex). NULL for Avito (not exposed).'; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/044_external_valuations_link.sql b/tradein-mvp/backend/data/sql/044_external_valuations_link.sql new file mode 100644 index 00000000..3d13ca94 --- /dev/null +++ b/tradein-mvp/backend/data/sql/044_external_valuations_link.sql @@ -0,0 +1,37 @@ +-- 044_external_valuations_link.sql +-- Purpose: Link external_valuations to canonical entities (house/listing) + add +-- sale band columns (low/high) per Cross_Source_Matching sec 13.1. +-- Table base in 026 + 029. Existing UNIQUE (source, cache_key) preserved. +-- Dependencies: 026_external_valuations.sql, 029_extend_matching_valuation_dynamics.sql, +-- 009_houses.sql, 002_core_tables.sql (listings) + +BEGIN; + +ALTER TABLE external_valuations + ADD COLUMN IF NOT EXISTS house_id bigint REFERENCES houses(id) ON DELETE CASCADE, + ADD COLUMN IF NOT EXISTS listing_id bigint REFERENCES listings(id) ON DELETE CASCADE, + ADD COLUMN IF NOT EXISTS low_price bigint, + ADD COLUMN IF NOT EXISTS high_price bigint; + +CREATE INDEX IF NOT EXISTS external_valuations_house_idx + ON external_valuations (house_id) + WHERE house_id IS NOT NULL; + +CREATE INDEX IF NOT EXISTS external_valuations_listing_idx + ON external_valuations (listing_id) + WHERE listing_id IS NOT NULL; + +CREATE UNIQUE INDEX IF NOT EXISTS external_valuations_canonical_uniq_idx + ON external_valuations (source, house_id, listing_id) + WHERE house_id IS NOT NULL AND listing_id IS NOT NULL; + +COMMENT ON COLUMN external_valuations.house_id IS + 'FK to canonical house (post-matching). NULL for pre-matching cache rows keyed by cache_key.'; +COMMENT ON COLUMN external_valuations.listing_id IS + 'FK to canonical listing (post-matching). NULL for house-level estimates.'; +COMMENT ON COLUMN external_valuations.low_price IS + 'Lower bound of sale band (Cian sometimes returns; NULL for Avito IMV).'; +COMMENT ON COLUMN external_valuations.high_price IS + 'Upper bound of sale band.'; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/045_house_placement_history_extend.sql b/tradein-mvp/backend/data/sql/045_house_placement_history_extend.sql new file mode 100644 index 00000000..cd9f4c5f --- /dev/null +++ b/tradein-mvp/backend/data/sql/045_house_placement_history_extend.sql @@ -0,0 +1,27 @@ +-- 045_house_placement_history_extend.sql +-- Purpose: Add source_confidence + notes + per-house 3-col UNIQUE per Cross_Source sec 14.1. +-- Table base in 017; source/ext_item_id already present. +-- Note: plan calls the external-id column "ext_listing_id", but existing +-- schema uses "ext_item_id" — semantic alias documented in COMMENT. +-- Dependencies: 017_house_placement_history.sql, 032_yandex_history.sql + +BEGIN; + +ALTER TABLE house_placement_history + ADD COLUMN IF NOT EXISTS source_confidence real DEFAULT 1.0, + ADD COLUMN IF NOT EXISTS notes text; + +CREATE UNIQUE INDEX IF NOT EXISTS hph_house_source_extid_uniq_idx + ON house_placement_history (house_id, source, ext_item_id) + WHERE house_id IS NOT NULL; + +COMMENT ON COLUMN house_placement_history.source_confidence IS + 'Confidence (0-1) that this history row belongs to the linked house. ' + '1.0 for direct Avito Houses Catalog / Cian offer; lower for Yandex Valuation synthetic IDs.'; +COMMENT ON COLUMN house_placement_history.notes IS + 'Free-form notes — used by dedup logic (e.g. ''dup_of:cian_offer:123''). ' + 'See Cross_Source_Matching_Strategy sec 14.3.'; +COMMENT ON COLUMN house_placement_history.ext_item_id IS + 'External listing id (alias of "ext_listing_id" in Master Plan sec 14.1).'; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/046_views.sql b/tradein-mvp/backend/data/sql/046_views.sql new file mode 100644 index 00000000..73b6aef1 --- /dev/null +++ b/tradein-mvp/backend/data/sql/046_views.sql @@ -0,0 +1,100 @@ +-- 046_views.sql +-- Purpose: Analytical / monitoring views for Phase 1.1. +-- - v_price_divergence (Cross_Source_Matching sec 2.2 + 4.6) — cross-source price disagreement +-- - v_cross_source_health (Master Plan sec 8.4) — listing-level spread metrics +-- - v_data_quality (Master Plan sec 8.1) — KPI dashboard rows +-- Dependencies: house_sources, listing_sources, houses, listings, house_placement_history. +-- All CREATE OR REPLACE — idempotent. + +BEGIN; + +-- ───────────────────────────────────────────────────────────────── +-- v_price_divergence — Cross_Source sec 2.2: 5%+ spread between sources for same canonical listing +-- ───────────────────────────────────────────────────────────────── +CREATE OR REPLACE VIEW v_price_divergence AS +SELECT + listing_id, + ARRAY_AGG(DISTINCT ext_source) AS sources, + MAX(price_rub) - MIN(price_rub) AS price_spread, + (MAX(price_rub) - MIN(price_rub))::float / NULLIF(MIN(price_rub), 0) * 100 AS spread_pct, + COUNT(*) AS source_count +FROM listing_sources +WHERE price_rub IS NOT NULL +GROUP BY listing_id +HAVING (MAX(price_rub) - MIN(price_rub))::float / NULLIF(MIN(price_rub), 0) > 0.05; + +COMMENT ON VIEW v_price_divergence IS + 'Listings with cross-source price spread > 5%. Used by quality monitor + manual review queue.'; + +-- ───────────────────────────────────────────────────────────────── +-- v_cross_source_health — Master Plan sec 8.4: listing-level spread breakdown +-- ───────────────────────────────────────────────────────────────── +CREATE OR REPLACE VIEW v_cross_source_health AS +SELECT + listing_id, + array_agg(ext_source ORDER BY price_rub) AS sources, + array_agg(price_rub ORDER BY price_rub) AS prices, + max(price_rub) - min(price_rub) AS spread, + (max(price_rub) - min(price_rub))::float + / NULLIF(min(price_rub), 0) * 100 AS spread_pct, + count(*) AS source_count +FROM listing_sources +WHERE price_rub IS NOT NULL +GROUP BY listing_id +HAVING count(*) >= 2; + +COMMENT ON VIEW v_cross_source_health IS + 'Per-listing cross-source price aggregation. Alert if spread_pct > 15% on > 100 listings.'; + +-- ───────────────────────────────────────────────────────────────── +-- v_data_quality — Master Plan sec 8.1: single-row KPI snapshot +-- ───────────────────────────────────────────────────────────────── +CREATE OR REPLACE VIEW v_data_quality AS +WITH active_listings AS ( + SELECT * FROM listings WHERE is_active = true +) +SELECT + (SELECT count(*) FROM houses) AS houses_total, + (SELECT count(*) FROM houses h + WHERE EXISTS (SELECT 1 FROM house_sources hs WHERE hs.house_id = h.id)) AS houses_with_source, + (SELECT count(*) FROM houses h + WHERE EXISTS (SELECT 1 FROM house_sources hs + WHERE hs.house_id = h.id AND hs.ext_source = 'avito')) AS houses_with_avito, + (SELECT count(*) FROM houses h + WHERE EXISTS (SELECT 1 FROM house_sources hs + WHERE hs.house_id = h.id AND hs.ext_source LIKE 'cian%')) AS houses_with_cian, + (SELECT count(*) FROM houses h + WHERE EXISTS (SELECT 1 FROM house_sources hs + WHERE hs.house_id = h.id AND hs.ext_source = 'yandex')) AS houses_with_yandex, + (SELECT count(*) FROM ( + SELECT house_id FROM house_sources GROUP BY house_id HAVING count(*) >= 2 + ) sub) AS houses_2plus_sources, + (SELECT count(*) FROM ( + SELECT house_id FROM house_sources GROUP BY house_id HAVING count(*) >= 3 + ) sub) AS houses_3plus_sources, + (SELECT count(*) FROM active_listings) AS listings_active, + (SELECT count(*) FROM ( + SELECT listing_id FROM listing_sources + WHERE listing_id IN (SELECT id FROM active_listings) + GROUP BY listing_id HAVING count(*) >= 2 + ) sub) AS listings_dedup_2sources, + (SELECT count(*) FROM active_listings WHERE lat IS NOT NULL) * 100.0 + / NULLIF((SELECT count(*) FROM active_listings), 0) AS pct_geocoded, + (SELECT count(*) FROM active_listings WHERE kadastr_num IS NOT NULL) * 100.0 + / NULLIF((SELECT count(*) FROM active_listings), 0) AS pct_cadastr, + (SELECT count(*) FROM active_listings WHERE description IS NOT NULL) * 100.0 + / NULLIF((SELECT count(*) FROM active_listings), 0) AS pct_description, + (SELECT count(*) FROM active_listings l + JOIN houses h ON h.id = l.house_id_fk + WHERE h.year_built IS NOT NULL) * 100.0 + / NULLIF((SELECT count(*) FROM active_listings), 0) AS pct_year_built, + NOW() - (SELECT max(scraped_at) FROM listings WHERE source = 'avito') AS avito_last_scrape_ago, + NOW() - (SELECT max(scraped_at) FROM listings WHERE source = 'cian') AS cian_last_scrape_ago, + NOW() - (SELECT max(scraped_at) FROM listings WHERE source = 'yandex') AS yandex_last_scrape_ago, + (SELECT count(*) FROM v_price_divergence) AS price_disagreements_count, + (SELECT count(*) FROM listings WHERE is_outlier = true) AS outliers_flagged; + +COMMENT ON VIEW v_data_quality IS + 'KPI snapshot. Refreshed on-demand by /api/v1/admin/data-quality endpoint (Master Plan sec 8.1).'; + +COMMIT; -- 2.45.3