d019b5327e
Multi-Source Integration Phase 1.1 thin-delta migrations. Gap audit showed 80%+ of planned schema already in 014/017/020/026/028/029/030/031 — this PR adds only the missing pieces: - 040 houses_extend: complex_id + avito_validated_at + cian_validated_at - 041 house_sources_noop: COMMENT aliases for ext_source/ext_id/matched_method - 042 listing_sources_price_divergence_idx: partial index for /search - 043 house_reviews_extend: likes column - 044 external_valuations_link: house_id/listing_id FKs + low/high price band - 045 house_placement_history_extend: source_confidence + notes + 3-col UNIQUE - 046 views: v_price_divergence + v_cross_source_health + v_data_quality All BEGIN/COMMIT, IF NOT EXISTS, FK ON DELETE CASCADE. Idempotent — safe re-apply on prod. Refs Master Plan sec 1, 4-8 + Cross_Source_Matching sec 2-4.
100 lines
6.3 KiB
PL/PgSQL
100 lines
6.3 KiB
PL/PgSQL
-- 046_views.sql
|
|
-- Purpose: Analytical / monitoring views for Phase 1.1.
|
|
-- - v_price_divergence (Cross_Source_Matching sec 2.2 + 4.6) — cross-source price disagreement
|
|
-- - v_cross_source_health (Master Plan sec 8.4) — listing-level spread metrics
|
|
-- - v_data_quality (Master Plan sec 8.1) — KPI dashboard rows
|
|
-- Dependencies: house_sources, listing_sources, houses, listings, house_placement_history.
|
|
-- All CREATE OR REPLACE — idempotent.
|
|
|
|
BEGIN;
|
|
|
|
-- ─────────────────────────────────────────────────────────────────
|
|
-- v_price_divergence — Cross_Source sec 2.2: 5%+ spread between sources for same canonical listing
|
|
-- ─────────────────────────────────────────────────────────────────
|
|
CREATE OR REPLACE VIEW v_price_divergence AS
|
|
SELECT
|
|
listing_id,
|
|
ARRAY_AGG(DISTINCT ext_source) AS sources,
|
|
MAX(price_rub) - MIN(price_rub) AS price_spread,
|
|
(MAX(price_rub) - MIN(price_rub))::float / NULLIF(MIN(price_rub), 0) * 100 AS spread_pct,
|
|
COUNT(*) AS source_count
|
|
FROM listing_sources
|
|
WHERE price_rub IS NOT NULL
|
|
GROUP BY listing_id
|
|
HAVING (MAX(price_rub) - MIN(price_rub))::float / NULLIF(MIN(price_rub), 0) > 0.05;
|
|
|
|
COMMENT ON VIEW v_price_divergence IS
|
|
'Listings with cross-source price spread > 5%. Used by quality monitor + manual review queue.';
|
|
|
|
-- ─────────────────────────────────────────────────────────────────
|
|
-- v_cross_source_health — Master Plan sec 8.4: listing-level spread breakdown
|
|
-- ─────────────────────────────────────────────────────────────────
|
|
CREATE OR REPLACE VIEW v_cross_source_health AS
|
|
SELECT
|
|
listing_id,
|
|
array_agg(ext_source ORDER BY price_rub) AS sources,
|
|
array_agg(price_rub ORDER BY price_rub) AS prices,
|
|
max(price_rub) - min(price_rub) AS spread,
|
|
(max(price_rub) - min(price_rub))::float
|
|
/ NULLIF(min(price_rub), 0) * 100 AS spread_pct,
|
|
count(*) AS source_count
|
|
FROM listing_sources
|
|
WHERE price_rub IS NOT NULL
|
|
GROUP BY listing_id
|
|
HAVING count(*) >= 2;
|
|
|
|
COMMENT ON VIEW v_cross_source_health IS
|
|
'Per-listing cross-source price aggregation. Alert if spread_pct > 15% on > 100 listings.';
|
|
|
|
-- ─────────────────────────────────────────────────────────────────
|
|
-- v_data_quality — Master Plan sec 8.1: single-row KPI snapshot
|
|
-- ─────────────────────────────────────────────────────────────────
|
|
CREATE OR REPLACE VIEW v_data_quality AS
|
|
WITH active_listings AS (
|
|
SELECT * FROM listings WHERE is_active = true
|
|
)
|
|
SELECT
|
|
(SELECT count(*) FROM houses) AS houses_total,
|
|
(SELECT count(*) FROM houses h
|
|
WHERE EXISTS (SELECT 1 FROM house_sources hs WHERE hs.house_id = h.id)) AS houses_with_source,
|
|
(SELECT count(*) FROM houses h
|
|
WHERE EXISTS (SELECT 1 FROM house_sources hs
|
|
WHERE hs.house_id = h.id AND hs.ext_source = 'avito')) AS houses_with_avito,
|
|
(SELECT count(*) FROM houses h
|
|
WHERE EXISTS (SELECT 1 FROM house_sources hs
|
|
WHERE hs.house_id = h.id AND hs.ext_source LIKE 'cian%')) AS houses_with_cian,
|
|
(SELECT count(*) FROM houses h
|
|
WHERE EXISTS (SELECT 1 FROM house_sources hs
|
|
WHERE hs.house_id = h.id AND hs.ext_source = 'yandex')) AS houses_with_yandex,
|
|
(SELECT count(*) FROM (
|
|
SELECT house_id FROM house_sources GROUP BY house_id HAVING count(*) >= 2
|
|
) sub) AS houses_2plus_sources,
|
|
(SELECT count(*) FROM (
|
|
SELECT house_id FROM house_sources GROUP BY house_id HAVING count(*) >= 3
|
|
) sub) AS houses_3plus_sources,
|
|
(SELECT count(*) FROM active_listings) AS listings_active,
|
|
(SELECT count(*) FROM (
|
|
SELECT listing_id FROM listing_sources
|
|
WHERE listing_id IN (SELECT id FROM active_listings)
|
|
GROUP BY listing_id HAVING count(*) >= 2
|
|
) sub) AS listings_dedup_2sources,
|
|
(SELECT count(*) FROM active_listings WHERE lat IS NOT NULL) * 100.0
|
|
/ NULLIF((SELECT count(*) FROM active_listings), 0) AS pct_geocoded,
|
|
(SELECT count(*) FROM active_listings WHERE kadastr_num IS NOT NULL) * 100.0
|
|
/ NULLIF((SELECT count(*) FROM active_listings), 0) AS pct_cadastr,
|
|
(SELECT count(*) FROM active_listings WHERE description IS NOT NULL) * 100.0
|
|
/ NULLIF((SELECT count(*) FROM active_listings), 0) AS pct_description,
|
|
(SELECT count(*) FROM active_listings l
|
|
JOIN houses h ON h.id = l.house_id_fk
|
|
WHERE h.year_built IS NOT NULL) * 100.0
|
|
/ NULLIF((SELECT count(*) FROM active_listings), 0) AS pct_year_built,
|
|
NOW() - (SELECT max(scraped_at) FROM listings WHERE source = 'avito') AS avito_last_scrape_ago,
|
|
NOW() - (SELECT max(scraped_at) FROM listings WHERE source = 'cian') AS cian_last_scrape_ago,
|
|
NOW() - (SELECT max(scraped_at) FROM listings WHERE source = 'yandex') AS yandex_last_scrape_ago,
|
|
(SELECT count(*) FROM v_price_divergence) AS price_disagreements_count,
|
|
(SELECT count(*) FROM listings WHERE is_outlier = true) AS outliers_flagged;
|
|
|
|
COMMENT ON VIEW v_data_quality IS
|
|
'KPI snapshot. Refreshed on-demand by /api/v1/admin/data-quality endpoint (Master Plan sec 8.1).';
|
|
|
|
COMMIT;
|