From 2296d33d28a1794326b169ba626701a403c52b01 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Thu, 2 Jul 2026 16:52:10 +0300 Subject: [PATCH] feat(market): contract views market.v_* over producer tables (#2130) --- .../data/sql/154_market_contract_views.sql | 219 ++++++++++++++++++ 1 file changed, 219 insertions(+) create mode 100644 tradein-mvp/backend/data/sql/154_market_contract_views.sql diff --git a/tradein-mvp/backend/data/sql/154_market_contract_views.sql b/tradein-mvp/backend/data/sql/154_market_contract_views.sql new file mode 100644 index 00000000..d7eb2769 --- /dev/null +++ b/tradein-mvp/backend/data/sql/154_market_contract_views.sql @@ -0,0 +1,219 @@ +-- 154_market_contract_views.sql +-- Public data contract for external consumers of the trade-in house dataset (#2130). +-- +-- Context: +-- `gendesign_reader` (101_gendesign_reader_role.sql) is currently granted SELECT +-- directly on 5 raw producer tables: houses, house_sources, houses_price_dynamics, +-- house_reliability_checks, house_reviews (102 grants listings/offer_price_history +-- separately). Consumers (gendesign-backend ETL #976, +-- backend/app/services/etl/newbuilding_crossload.py) read these raw tables directly, +-- which means every scraper-driven ALTER TABLE (new column, renamed column, dropped +-- bookkeeping field) is a silent breaking change for an external product. +-- +-- This migration introduces `market.v_*` views as a stable, explicit-column contract +-- layer over those 5 raw tables. Consumers should migrate to reading the views +-- instead of the raw tables. The column list below IS the promise of stability — +-- adding a column later is backward compatible, renaming/removing one is not. +-- +-- Regranting `gendesign_reader` to the views + revoking direct raw-table access is a +-- SEPARATE follow-up (#2138) — intentionally NOT done here. This migration only adds +-- the views; existing raw-table grants (101_gendesign_reader_role.sql) are untouched. +-- +-- Column selection: +-- Every column actually read by the current consumer +-- (backend/app/services/etl/newbuilding_crossload.py `_SOURCE_SQL`, gendesign repo) is +-- included: houses.{source, ext_house_id, url, slug, address, full_address, lat, lon, +-- house_class, developer_name, developer_key, year_built, total_floors, total_units, +-- rating, reviews_count, houses_by_turn, cian_internal_house_id, raw_payload, +-- last_scraped_at} + house_sources.{house_id, ext_source, ext_id} (LEFT JOIN for +-- yandex_jk_id). +-- +-- houses_price_dynamics / house_reliability_checks / house_reviews have no current +-- gendesign-side consumer, but are granted to gendesign_reader (101) as future ETL +-- sources — views expose their full content-bearing column set so future consumers +-- get the same stability guarantee from day one. +-- +-- Deliberately EXCLUDED from all views (producer-internal, high-churn, not part of +-- the public contract): +-- - raw_payload / raw_characteristics / map_pins / rating_distribution — raw +-- scrape-source blobs kept for internal retrofit, not a stable shape. +-- (houses.raw_payload IS however read by the ETL today — see v_houses below, +-- kept for that reason; house_sources/house_reviews raw_payload are NOT read +-- by any current consumer and are excluded.) +-- - *_validated_at / *_status / *_qc_* / *_error_reason / last_imv_attempt_at — +-- internal scrape-state bookkeeping (dadata QC codes, IMV backfill status), +-- churns independently of the actual house data. +-- - complex_id — unused FK placeholder (no complexes table yet, per 040 comment). +-- +-- Dependencies: +-- - 105_market_schema_yandex_enrichment.sql (CREATE SCHEMA market) +-- - 009_houses.sql + all subsequent houses ALTERs (010/020/029/031/040/064/070/071/105) +-- - 028_matching_tables.sql + 029_extend_matching_valuation_dynamics.sql (house_sources) +-- - 024_houses_price_dynamics.sql + 029 dims (houses_price_dynamics) +-- - 025_house_reliability_checks.sql (house_reliability_checks) +-- - 014_house_reviews.sql + 043_house_reviews_extend.sql (house_reviews) +-- +-- Idempotent: CREATE SCHEMA IF NOT EXISTS + CREATE OR REPLACE VIEW — safe to re-run. +-- AUTO-APPLIED on prod via deploy-tradein.yml/_schema_migrations. +-- Does NOT touch grants — gendesign_reader keeps reading raw tables until #2138. + +BEGIN; + +CREATE SCHEMA IF NOT EXISTS market; + +-- ── market.v_houses ────────────────────────────────────────────────────────── +CREATE OR REPLACE VIEW market.v_houses AS +SELECT + id, + source, + ext_house_id, + url, + slug, + address, + full_address, + short_address, + lat, + lon, + geom, + year_built, + house_type, + house_class, + material_walls, + material_floors, + series_name, + total_floors, + total_units, + entrances, + flat_count, + is_emergency, + passenger_elevators, + cargo_elevators, + has_concierge, + closed_yard, + has_playground, + hot_water, + heat_supply_type, + gas_supply_type, + overlap_type, + parking_type, + infrastructure_summary, + infrastructure_walk_distance, + developer_name, + developer_key, + management_company_id, + rating, + reviews_count, + rating_score, + rating_string, + transport_accessibility_rate, + has_panorama, + advantages, + banks, + builders, + houses_by_turn, + corpus_count, + commission_year, + commission_month, + total_area_ha, + cadastral_number, + house_fias_id, + yandex_jk_id, + yandex_jk_slug, + cian_internal_house_id, + cian_zhk_url, + raw_payload, + first_seen_at, + last_scraped_at +FROM public.houses; + +COMMENT ON VIEW market.v_houses IS + 'Stable public contract over public.houses (#2130). Explicit column list is the ' + 'stability promise — do not SELECT * against the base table from external ' + 'consumers. raw_payload is included because it is read today by gendesign ETL ' + '#976 (newbuilding_crossload.py); scraper-internal QC/status/validated_at ' + 'bookkeeping columns are intentionally excluded.'; + +-- ── market.v_house_sources ─────────────────────────────────────────────────── +CREATE OR REPLACE VIEW market.v_house_sources AS +SELECT + id, + house_id, + ext_source, + ext_id, + confidence, + matched_method, + matched_at, + ext_url, + last_seen_at +FROM public.house_sources; + +COMMENT ON VIEW market.v_house_sources IS + 'Stable public contract over public.house_sources (#2130). Cross-source house ' + 'match links (house_id -> ext_source/ext_id). raw_payload (internal source ' + 'snapshot for diffing) is intentionally excluded — not read by any current ' + 'consumer.'; + +-- ── market.v_houses_price_dynamics ─────────────────────────────────────────── +CREATE OR REPLACE VIEW market.v_houses_price_dynamics AS +SELECT + id, + house_id, + month_date, + price_per_sqm, + source, + room_count, + prices_type, + period, + recorded_at +FROM public.houses_price_dynamics; + +COMMENT ON VIEW market.v_houses_price_dynamics IS + 'Stable public contract over public.houses_price_dynamics (#2130). Monthly ' + 'price-per-sqm time series per house, dimensioned by room_count/prices_type/' + 'period. No current gendesign consumer — view exists for forward-compat with ' + 'the gendesign_reader grant in 101_gendesign_reader_role.sql.'; + +-- ── market.v_house_reliability_checks ──────────────────────────────────────── +CREATE OR REPLACE VIEW market.v_house_reliability_checks AS +SELECT + id, + house_id, + check_status, + check_name, + details, + source, + recorded_at +FROM public.house_reliability_checks; + +COMMENT ON VIEW market.v_house_reliability_checks IS + 'Stable public contract over public.house_reliability_checks (#2130). ' + 'наш.дом.рф reliability check results scraped via Cian newbuilding pages. ' + 'No current gendesign consumer — view exists for forward-compat with the ' + 'gendesign_reader grant in 101_gendesign_reader_role.sql.'; + +-- ── market.v_house_reviews ──────────────────────────────────────────────────── +CREATE OR REPLACE VIEW market.v_house_reviews AS +SELECT + id, + house_id, + source, + ext_review_id, + author_name, + review_title, + score, + model_experience, + rated_date, + text_main, + text_pros, + text_cons, + likes, + scraped_at +FROM public.house_reviews; + +COMMENT ON VIEW market.v_house_reviews IS + 'Stable public contract over public.house_reviews (#2130). Resident reviews ' + 'from Avito/Cian/Yandex Houses Catalog. raw_payload excluded (internal retrofit ' + 'blob, no current consumer). No current gendesign consumer — view exists for ' + 'forward-compat with the gendesign_reader grant in 101_gendesign_reader_role.sql.'; + +COMMIT;