feat(market): contract views market.v_* over producer tables (#2130)

tradein-mvp/backend/data/sql/154_market_contract_views.sql

@@ -0,0 +1,219 @@
+-- 154_market_contract_views.sql
+-- Public data contract for external consumers of the trade-in house dataset (#2130).
+--
+-- Context:
+--   `gendesign_reader` (101_gendesign_reader_role.sql) is currently granted SELECT
+--   directly on 5 raw producer tables: houses, house_sources, houses_price_dynamics,
+--   house_reliability_checks, house_reviews (102 grants listings/offer_price_history
+--   separately). Consumers (gendesign-backend ETL #976,
+--   backend/app/services/etl/newbuilding_crossload.py) read these raw tables directly,
+--   which means every scraper-driven ALTER TABLE (new column, renamed column, dropped
+--   bookkeeping field) is a silent breaking change for an external product.
+--
+--   This migration introduces `market.v_*` views as a stable, explicit-column contract
+--   layer over those 5 raw tables. Consumers should migrate to reading the views
+--   instead of the raw tables. The column list below IS the promise of stability —
+--   adding a column later is backward compatible, renaming/removing one is not.
+--
+--   Regranting `gendesign_reader` to the views + revoking direct raw-table access is a
+--   SEPARATE follow-up (#2138) — intentionally NOT done here. This migration only adds
+--   the views; existing raw-table grants (101_gendesign_reader_role.sql) are untouched.
+--
+-- Column selection:
+--   Every column actually read by the current consumer
+--   (backend/app/services/etl/newbuilding_crossload.py `_SOURCE_SQL`, gendesign repo) is
+--   included: houses.{source, ext_house_id, url, slug, address, full_address, lat, lon,
+--   house_class, developer_name, developer_key, year_built, total_floors, total_units,
+--   rating, reviews_count, houses_by_turn, cian_internal_house_id, raw_payload,
+--   last_scraped_at} + house_sources.{house_id, ext_source, ext_id} (LEFT JOIN for
+--   yandex_jk_id).
+--
+--   houses_price_dynamics / house_reliability_checks / house_reviews have no current
+--   gendesign-side consumer, but are granted to gendesign_reader (101) as future ETL
+--   sources — views expose their full content-bearing column set so future consumers
+--   get the same stability guarantee from day one.
+--
+--   Deliberately EXCLUDED from all views (producer-internal, high-churn, not part of
+--   the public contract):
+--     - raw_payload / raw_characteristics / map_pins / rating_distribution — raw
+--       scrape-source blobs kept for internal retrofit, not a stable shape.
+--       (houses.raw_payload IS however read by the ETL today — see v_houses below,
+--       kept for that reason; house_sources/house_reviews raw_payload are NOT read
+--       by any current consumer and are excluded.)
+--     - *_validated_at / *_status / *_qc_* / *_error_reason / last_imv_attempt_at —
+--       internal scrape-state bookkeeping (dadata QC codes, IMV backfill status),
+--       churns independently of the actual house data.
+--     - complex_id — unused FK placeholder (no complexes table yet, per 040 comment).
+--
+-- Dependencies:
+--   - 105_market_schema_yandex_enrichment.sql (CREATE SCHEMA market)
+--   - 009_houses.sql + all subsequent houses ALTERs (010/020/029/031/040/064/070/071/105)
+--   - 028_matching_tables.sql + 029_extend_matching_valuation_dynamics.sql (house_sources)
+--   - 024_houses_price_dynamics.sql + 029 dims (houses_price_dynamics)
+--   - 025_house_reliability_checks.sql (house_reliability_checks)
+--   - 014_house_reviews.sql + 043_house_reviews_extend.sql (house_reviews)
+--
+-- Idempotent: CREATE SCHEMA IF NOT EXISTS + CREATE OR REPLACE VIEW — safe to re-run.
+-- AUTO-APPLIED on prod via deploy-tradein.yml/_schema_migrations.
+-- Does NOT touch grants — gendesign_reader keeps reading raw tables until #2138.
+
+BEGIN;
+
+CREATE SCHEMA IF NOT EXISTS market;
+
+-- ── market.v_houses ──────────────────────────────────────────────────────────
+CREATE OR REPLACE VIEW market.v_houses AS
+SELECT
+    id,
+    source,
+    ext_house_id,
+    url,
+    slug,
+    address,
+    full_address,
+    short_address,
+    lat,
+    lon,
+    geom,
+    year_built,
+    house_type,
+    house_class,
+    material_walls,
+    material_floors,
+    series_name,
+    total_floors,
+    total_units,
+    entrances,
+    flat_count,
+    is_emergency,
+    passenger_elevators,
+    cargo_elevators,
+    has_concierge,
+    closed_yard,
+    has_playground,
+    hot_water,
+    heat_supply_type,
+    gas_supply_type,
+    overlap_type,
+    parking_type,
+    infrastructure_summary,
+    infrastructure_walk_distance,
+    developer_name,
+    developer_key,
+    management_company_id,
+    rating,
+    reviews_count,
+    rating_score,
+    rating_string,
+    transport_accessibility_rate,
+    has_panorama,
+    advantages,
+    banks,
+    builders,
+    houses_by_turn,
+    corpus_count,
+    commission_year,
+    commission_month,
+    total_area_ha,
+    cadastral_number,
+    house_fias_id,
+    yandex_jk_id,
+    yandex_jk_slug,
+    cian_internal_house_id,
+    cian_zhk_url,
+    raw_payload,
+    first_seen_at,
+    last_scraped_at
+FROM public.houses;
+
+COMMENT ON VIEW market.v_houses IS
+    'Stable public contract over public.houses (#2130). Explicit column list is the '
+    'stability promise — do not SELECT * against the base table from external '
+    'consumers. raw_payload is included because it is read today by gendesign ETL '
+    '#976 (newbuilding_crossload.py); scraper-internal QC/status/validated_at '
+    'bookkeeping columns are intentionally excluded.';
+
+-- ── market.v_house_sources ───────────────────────────────────────────────────
+CREATE OR REPLACE VIEW market.v_house_sources AS
+SELECT
+    id,
+    house_id,
+    ext_source,
+    ext_id,
+    confidence,
+    matched_method,
+    matched_at,
+    ext_url,
+    last_seen_at
+FROM public.house_sources;
+
+COMMENT ON VIEW market.v_house_sources IS
+    'Stable public contract over public.house_sources (#2130). Cross-source house '
+    'match links (house_id -> ext_source/ext_id). raw_payload (internal source '
+    'snapshot for diffing) is intentionally excluded — not read by any current '
+    'consumer.';
+
+-- ── market.v_houses_price_dynamics ───────────────────────────────────────────
+CREATE OR REPLACE VIEW market.v_houses_price_dynamics AS
+SELECT
+    id,
+    house_id,
+    month_date,
+    price_per_sqm,
+    source,
+    room_count,
+    prices_type,
+    period,
+    recorded_at
+FROM public.houses_price_dynamics;
+
+COMMENT ON VIEW market.v_houses_price_dynamics IS
+    'Stable public contract over public.houses_price_dynamics (#2130). Monthly '
+    'price-per-sqm time series per house, dimensioned by room_count/prices_type/'
+    'period. No current gendesign consumer — view exists for forward-compat with '
+    'the gendesign_reader grant in 101_gendesign_reader_role.sql.';
+
+-- ── market.v_house_reliability_checks ────────────────────────────────────────
+CREATE OR REPLACE VIEW market.v_house_reliability_checks AS
+SELECT
+    id,
+    house_id,
+    check_status,
+    check_name,
+    details,
+    source,
+    recorded_at
+FROM public.house_reliability_checks;
+
+COMMENT ON VIEW market.v_house_reliability_checks IS
+    'Stable public contract over public.house_reliability_checks (#2130). '
+    'наш.дом.рф reliability check results scraped via Cian newbuilding pages. '
+    'No current gendesign consumer — view exists for forward-compat with the '
+    'gendesign_reader grant in 101_gendesign_reader_role.sql.';
+
+-- ── market.v_house_reviews ────────────────────────────────────────────────────
+CREATE OR REPLACE VIEW market.v_house_reviews AS
+SELECT
+    id,
+    house_id,
+    source,
+    ext_review_id,
+    author_name,
+    review_title,
+    score,
+    model_experience,
+    rated_date,
+    text_main,
+    text_pros,
+    text_cons,
+    likes,
+    scraped_at
+FROM public.house_reviews;
+
+COMMENT ON VIEW market.v_house_reviews IS
+    'Stable public contract over public.house_reviews (#2130). Resident reviews '
+    'from Avito/Cian/Yandex Houses Catalog. raw_payload excluded (internal retrofit '
+    'blob, no current consumer). No current gendesign consumer — view exists for '
+    'forward-compat with the gendesign_reader grant in 101_gendesign_reader_role.sql.';
+
+COMMIT;