191 lines
7.7 KiB
PL/PgSQL
191 lines
7.7 KiB
PL/PgSQL
-- 050_search_optimization.sql
|
|
-- Purpose: Search-time performance optimization for cross-source listings.
|
|
-- Adds pg_trgm + generated tsvector + 10 indexes (master plan sec 5.1)
|
|
-- + listings_search_mv materialized view (master plan sec 6.5)
|
|
-- + 5 matview indexes.
|
|
-- Dependencies: 002_core_tables.sql (listings), 009_houses.sql, 028_matching_tables.sql,
|
|
-- 030_listings_alter_yandex.sql.
|
|
-- Deploy order: Apply after 046_views.sql.
|
|
-- Re-run safe: all ADD COLUMN / CREATE INDEX / CREATE MATERIALIZED VIEW use IF NOT EXISTS.
|
|
--
|
|
-- Sources:
|
|
-- Multi-Source Integration Master Plan sec 5.1 (10 индексов on listings)
|
|
-- Multi-Source Integration Master Plan sec 6.5 (listings_search_mv)
|
|
|
|
BEGIN;
|
|
|
|
-- =============================================================================
|
|
-- Extensions
|
|
-- =============================================================================
|
|
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
|
|
|
-- =============================================================================
|
|
-- Generated tsvector column for full-text search (russian config)
|
|
-- Master plan sec 5.1 index #6 + sec 6.5 matview tsv definition.
|
|
-- Weight A → description, weight B → address.
|
|
-- =============================================================================
|
|
DO $$
|
|
BEGIN
|
|
IF NOT EXISTS (
|
|
SELECT 1 FROM information_schema.columns
|
|
WHERE table_name = 'listings' AND column_name = 'tsv'
|
|
) THEN
|
|
ALTER TABLE listings ADD COLUMN tsv tsvector
|
|
GENERATED ALWAYS AS (
|
|
setweight(to_tsvector('russian', coalesce(description, '')), 'A')
|
|
|| setweight(to_tsvector('russian', coalesce(address, '')), 'B')
|
|
) STORED;
|
|
END IF;
|
|
END
|
|
$$;
|
|
|
|
-- =============================================================================
|
|
-- 10 indexes per master plan sec 5.1
|
|
-- =============================================================================
|
|
|
|
-- 1) Hot path: active listings by geo radius (GIST partial)
|
|
CREATE INDEX IF NOT EXISTS listings_geom_active_idx
|
|
ON listings USING GIST (geom)
|
|
WHERE is_active = true;
|
|
|
|
-- 2) Active + filter composite: rooms + price_rub + area_m2 + scraped_at DESC
|
|
CREATE INDEX IF NOT EXISTS listings_active_filter_idx
|
|
ON listings (rooms, price_rub, area_m2, scraped_at DESC)
|
|
WHERE is_active = true;
|
|
|
|
-- 3) House + price (price range within a house)
|
|
CREATE INDEX IF NOT EXISTS listings_house_price_idx
|
|
ON listings (house_id_fk, price_rub)
|
|
WHERE is_active = true;
|
|
|
|
-- 4) Recency sort (scraped_at DESC)
|
|
CREATE INDEX IF NOT EXISTS listings_scraped_desc_idx
|
|
ON listings (scraped_at DESC)
|
|
WHERE is_active = true;
|
|
|
|
-- 5) Address fuzzy search (trigram)
|
|
CREATE INDEX IF NOT EXISTS listings_address_trgm_idx
|
|
ON listings USING GIN (address gin_trgm_ops)
|
|
WHERE address IS NOT NULL;
|
|
|
|
-- 6) Full-text search (tsv column above)
|
|
CREATE INDEX IF NOT EXISTS listings_tsv_idx
|
|
ON listings USING GIN (tsv);
|
|
|
|
-- 7) Cadastral exact lookup (partial)
|
|
CREATE INDEX IF NOT EXISTS listings_kadastr_idx
|
|
ON listings (kadastr_num)
|
|
WHERE kadastr_num IS NOT NULL;
|
|
|
|
-- 8) listing_sources (listing_id) — REMOVED in PR 7
|
|
-- duplicate of 028_matching_tables.sql's listing_sources_listing_idx.
|
|
|
|
-- 9) listing_sources (listing_id, price_rub) — for price divergence detection
|
|
CREATE INDEX IF NOT EXISTS listing_sources_price_divergence_idx2
|
|
ON listing_sources (listing_id, price_rub)
|
|
WHERE price_rub IS NOT NULL;
|
|
|
|
-- 10) houses fingerprint + kadastr (consolidated houses indexes)
|
|
-- Note: houses_geom_idx2 removed in PR 7 — duplicated 009_houses.sql's houses_geom_idx.
|
|
CREATE INDEX IF NOT EXISTS houses_fingerprint_idx2
|
|
ON houses (address_fingerprint)
|
|
WHERE address_fingerprint IS NOT NULL;
|
|
CREATE INDEX IF NOT EXISTS houses_kadastr_idx2
|
|
ON houses (cadastral_number)
|
|
WHERE cadastral_number IS NOT NULL;
|
|
|
|
COMMIT;
|
|
|
|
-- =============================================================================
|
|
-- Materialized view: listings_search_mv (master plan sec 6.5)
|
|
-- Separate BEGIN/COMMIT block — matview creation can be slow on populated tables.
|
|
-- =============================================================================
|
|
BEGIN;
|
|
|
|
DROP MATERIALIZED VIEW IF EXISTS listings_search_mv;
|
|
|
|
CREATE MATERIALIZED VIEW listings_search_mv AS
|
|
SELECT
|
|
l.id AS listing_id,
|
|
l.source,
|
|
l.source_url,
|
|
l.address,
|
|
l.geom,
|
|
l.lat,
|
|
l.lon AS lng,
|
|
l.rooms,
|
|
l.area_m2 AS total_area,
|
|
l.floor,
|
|
l.total_floors,
|
|
l.price_rub,
|
|
l.price_per_m2,
|
|
l.kadastr_num,
|
|
l.is_active,
|
|
l.scraped_at,
|
|
-- House denorm
|
|
h.id AS house_id,
|
|
h.year_built,
|
|
h.house_class,
|
|
h.developer_name,
|
|
h.rating AS house_rating,
|
|
h.reviews_count AS house_ratings_count,
|
|
-- Cross-source aggregates
|
|
(SELECT count(*) FROM listing_sources ls WHERE ls.listing_id = l.id) AS source_count,
|
|
(SELECT array_agg(DISTINCT ext_source) FROM listing_sources ls WHERE ls.listing_id = l.id) AS sources,
|
|
(SELECT bool_or(ext_source = 'avito') FROM listing_sources ls WHERE ls.listing_id = l.id) AS has_avito,
|
|
(SELECT bool_or(ext_source = 'cian') FROM listing_sources ls WHERE ls.listing_id = l.id) AS has_cian,
|
|
(SELECT bool_or(ext_source = 'yandex_realty') FROM listing_sources ls WHERE ls.listing_id = l.id) AS has_yandex,
|
|
-- Price percentile within house
|
|
(SELECT percentile_cont(0.5) WITHIN GROUP (ORDER BY ll.price_per_m2)
|
|
FROM listings ll
|
|
WHERE ll.house_id_fk = l.house_id_fk AND ll.is_active = true) AS house_median_ppm2,
|
|
-- DEFAULT: district / distance_to_metro_m / last_price_change / photos_count
|
|
-- not present in current schema — placeholders. (vault sec 6.5 doesn't define them either;
|
|
-- task description listed them but underlying columns don't exist yet — populate post-Phase 4.)
|
|
NULL::text AS district,
|
|
NULL::int AS distance_to_metro_m,
|
|
NULL::timestamptz AS last_price_change,
|
|
NULL::int AS photos_count,
|
|
-- Trigram-ready columns
|
|
l.address AS address_trgm,
|
|
-- Aggregated tsv (description + address + developer_name)
|
|
to_tsvector('russian',
|
|
coalesce(l.description, '') || ' ' ||
|
|
coalesce(l.address, '') || ' ' ||
|
|
coalesce(h.developer_name, '')
|
|
) AS tsv
|
|
FROM listings l
|
|
LEFT JOIN houses h ON h.id = l.house_id_fk
|
|
WHERE l.is_active = true
|
|
AND COALESCE(l.canonical, true) = true;
|
|
|
|
-- =============================================================================
|
|
-- 5 matview indexes per master plan sec 6.5
|
|
-- =============================================================================
|
|
CREATE UNIQUE INDEX listings_search_mv_id_idx
|
|
ON listings_search_mv (listing_id);
|
|
|
|
CREATE INDEX listings_search_mv_geom_idx
|
|
ON listings_search_mv USING GIST (geom);
|
|
|
|
CREATE INDEX listings_search_mv_filters_idx
|
|
ON listings_search_mv (rooms, price_rub, total_area, scraped_at DESC);
|
|
|
|
CREATE INDEX listings_search_mv_address_trgm_idx
|
|
ON listings_search_mv USING GIN (address_trgm gin_trgm_ops);
|
|
|
|
CREATE INDEX listings_search_mv_tsv_idx
|
|
ON listings_search_mv USING GIN (tsv);
|
|
|
|
CREATE INDEX listings_search_mv_sources_idx
|
|
ON listings_search_mv (has_avito, has_cian, has_yandex);
|
|
|
|
COMMIT;
|
|
|
|
-- =============================================================================
|
|
-- Initial populate: REFRESH CONCURRENTLY would fail (matview just created — never populated).
|
|
-- First populate is implicit via CREATE MATERIALIZED VIEW AS SELECT ... above.
|
|
-- Subsequent refreshes (Celery task) use CONCURRENTLY.
|
|
-- Skip explicit REFRESH here — empty source data scenarios will result in empty matview but
|
|
-- creation is still valid.
|
|
-- =============================================================================
|