From 65c2824507c11166c1fd903e8f40953273f7c1c4 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 15:53:11 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20SQL=20migrations=20030-032=20?= =?UTF-8?q?=E2=80=94=20Yandex=20schema=20extensions?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 030_listings_alter_yandex.sql: 7 cols (yandex_offer_id, views_total_yandex, publish_date_relative, sale_type_text, agency_name/founded_year/objects_count) + 2 partial indexes (yandex_offer_id, agency_name) - 031_houses_alter_yandex.sql: 9 cols (yandex_jk_id/slug, commission_year/month, total_area_ha, corpus_count, yandex_validated_at, yandex_total_listings, has_panorama) + 2 partial indexes (yandex_jk_id, commission_year) - 032_yandex_history.sql: enables source='yandex_valuation' on existing house_placement_history + 2 indexes (hph_source_idx, hph_house_source_idx) Foundation for YandexRealtyScraper v1 (Stages 2-9 follow on subsequent PRs). Note: numbered 030-032 because 029 was already used by PR #448 (matching/ valuation/dynamics extensions). All BEGIN/COMMIT, ADD COLUMN IF NOT EXISTS, nullable, idempotent. --- .../data/sql/030_listings_alter_yandex.sql | 45 +++++++++++++++++++ .../data/sql/031_houses_alter_yandex.sql | 41 +++++++++++++++++ .../backend/data/sql/032_yandex_history.sql | 31 +++++++++++++ 3 files changed, 117 insertions(+) create mode 100644 tradein-mvp/backend/data/sql/030_listings_alter_yandex.sql create mode 100644 tradein-mvp/backend/data/sql/031_houses_alter_yandex.sql create mode 100644 tradein-mvp/backend/data/sql/032_yandex_history.sql diff --git a/tradein-mvp/backend/data/sql/030_listings_alter_yandex.sql b/tradein-mvp/backend/data/sql/030_listings_alter_yandex.sql new file mode 100644 index 00000000..3fbed965 --- /dev/null +++ b/tradein-mvp/backend/data/sql/030_listings_alter_yandex.sql @@ -0,0 +1,45 @@ +-- 030_listings_alter_yandex.sql +-- Purpose: Add Yandex Realty-specific columns to listings table (SERP + detail fields). +-- Dependencies: listings table created in 002_core_tables.sql / 011_listings_alter.sql +-- 019_listings_alter_cian.sql already adds views_total (int) for Cian. +-- Yandex gets a parallel column views_total_yandex to preserve source attribution — +-- a single shared column cannot distinguish which scraper wrote the value when both +-- are active concurrently (conflict-resolution ambiguity). +-- Deploy order: Apply after 029_extend_matching_valuation_dynamics.sql +-- +-- Sources: +-- - decisions/Schema_YandexRealty_Recon.md sec 10.1 (SERP offer card) +-- - decisions/Schema_YandexRealty_Recon.md sec 12.3 (OfferCardAuthorInfo agency block) +-- - YandexRealtyScraper_v1_Implementation_Plan Stage 1 +-- +-- Idempotent: all ALTER TABLE uses ADD COLUMN IF NOT EXISTS; all CREATE INDEX use IF NOT EXISTS. + +BEGIN; + +ALTER TABLE listings + -- Yandex Realty offer identification + ADD COLUMN IF NOT EXISTS yandex_offer_id text, -- numeric string (e.g. '7567094292504417257'), kept as text + + -- Yandex SERP/detail stats + ADD COLUMN IF NOT EXISTS views_total_yandex int, -- "N просмотров" из OfferCardSummary; views_total already used by Cian + ADD COLUMN IF NOT EXISTS publish_date_relative text, -- "6 часов назад" / "вчера" / "N дней назад" raw text + + -- Yandex sale type (text form differs from Cian sale_type enum) + ADD COLUMN IF NOT EXISTS sale_type_text text, -- 'свободная продажа' / 'альтернативная' (raw RU phrase) + + -- Yandex agency block (OfferCardAuthorInfo) + ADD COLUMN IF NOT EXISTS agency_name text, -- 'Агентство «Диал»' + ADD COLUMN IF NOT EXISTS agency_founded_year int, -- "Год основания N" parsed + ADD COLUMN IF NOT EXISTS agency_objects_count int; -- "N объектов" parsed + +-- Partial index on yandex_offer_id for fast lookup by Yandex identifier +CREATE INDEX IF NOT EXISTS listings_yandex_offer_id_idx + ON listings (yandex_offer_id) + WHERE yandex_offer_id IS NOT NULL; + +-- Partial index on agency_name for analytics queries filtered by agency +CREATE INDEX IF NOT EXISTS listings_agency_name_idx + ON listings (agency_name) + WHERE agency_name IS NOT NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/031_houses_alter_yandex.sql b/tradein-mvp/backend/data/sql/031_houses_alter_yandex.sql new file mode 100644 index 00000000..66885b72 --- /dev/null +++ b/tradein-mvp/backend/data/sql/031_houses_alter_yandex.sql @@ -0,0 +1,41 @@ +-- 031_houses_alter_yandex.sql +-- Purpose: Add Yandex Realty-specific columns to houses table (ЖК landing + valuation enrichment). +-- Dependencies: +-- - houses table: 009_houses.sql +-- - 020_houses_alter_cian.sql (prior Cian columns must exist; no FK dependency, column-level only) +-- Deploy order: Apply after 030_listings_alter_yandex.sql +-- +-- Sources: +-- - decisions/Schema_YandexRealty_Recon.md sec 11.2 (Newbuilding ЖК landing fields) +-- - decisions/Schema_YandexRealty_Recon.md sec 13.1 (Yandex Valuation tool enrichment) +-- - YandexRealtyScraper_v1_Implementation_Plan Stage 1 +-- +-- Idempotent: all ALTER TABLE uses ADD COLUMN IF NOT EXISTS; all CREATE INDEX use IF NOT EXISTS. + +BEGIN; + +ALTER TABLE houses + -- Yandex Newbuilding (ЖК) landing + ADD COLUMN IF NOT EXISTS yandex_jk_id text, -- '1592987' + ADD COLUMN IF NOT EXISTS yandex_jk_slug text, -- 'tatlin' + ADD COLUMN IF NOT EXISTS commission_year int, -- 2023 + ADD COLUMN IF NOT EXISTS commission_month text, -- 'июнь' (raw RU month) + ADD COLUMN IF NOT EXISTS total_area_ha numeric(6,2), -- 1.50 + ADD COLUMN IF NOT EXISTS corpus_count int, -- 3 (parsed from "три башни") + + -- Yandex Valuation tool enrichment + ADD COLUMN IF NOT EXISTS yandex_validated_at timestamptz, -- last enrichment timestamp + ADD COLUMN IF NOT EXISTS yandex_total_listings int, -- "N объектов" в истории дома + ADD COLUMN IF NOT EXISTS has_panorama boolean; -- Yandex Panorama 3D view available + +-- Partial index on yandex_jk_id for fast lookup by Yandex newbuilding identifier +CREATE INDEX IF NOT EXISTS houses_yandex_jk_idx + ON houses (yandex_jk_id) + WHERE yandex_jk_id IS NOT NULL; + +-- Partial index on commission_year for filtering by year of commissioning +CREATE INDEX IF NOT EXISTS houses_commission_year_idx + ON houses (commission_year) + WHERE commission_year IS NOT NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/032_yandex_history.sql b/tradein-mvp/backend/data/sql/032_yandex_history.sql new file mode 100644 index 00000000..5d4355b5 --- /dev/null +++ b/tradein-mvp/backend/data/sql/032_yandex_history.sql @@ -0,0 +1,31 @@ +-- 032_yandex_history.sql +-- Purpose: Enable source='yandex_valuation' on existing house_placement_history table by +-- updating the column COMMENT to document the new value and adding analytics-friendly +-- indexes for efficient filtered queries on the new source. +-- Dependencies: 017_house_placement_history.sql (table must already exist) +-- Deploy order: Apply after 031_houses_alter_yandex.sql +-- +-- Sources: +-- - decisions/Schema_YandexRealty_Recon.md sec 12.5 + 13.2 +-- - YandexRealtyScraper_v1_Implementation_Plan Stage 1 +-- +-- Note: Idempotent — does NOT alter table schema (no ADD/DROP COLUMN). +-- Only updates COMMENT metadata and creates indexes with IF NOT EXISTS. +-- Safe to re-run; COMMENT ON COLUMN is always an overwrite (no-error on repeat). + +BEGIN; + +-- Update column comment to document the new source enum value +COMMENT ON COLUMN house_placement_history.source IS + 'Source of history record: avito / avito_imv / cian / yandex_valuation'; + +-- Partial index for analytics queries filtered by source (mostly yandex_valuation) +CREATE INDEX IF NOT EXISTS hph_source_idx + ON house_placement_history (source); + +-- Composite index for "all history for a house from a specific source" +CREATE INDEX IF NOT EXISTS hph_house_source_idx + ON house_placement_history (house_id, source, last_price_date DESC) + WHERE house_id IS NOT NULL; + +COMMIT;