From 1ad2565af1ff77c4ce1258f02cb1e4bc66ebdaf4 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 15:14:49 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20SQL=20migrations=20019-028=20?= =?UTF-8?q?=E2=80=94=20Cian=20schema=20+=20matching=20+=20cookies?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 019: ALTER listings (windows_view_type, wcs counts, ceiling_height, repair_type, views, living_area, bedrooms, balconies, loggias, description_minhash, cadastral_number, phones, bargain_allowed, sale_type, house_source/ext_id, metro_stations) - 020: ALTER houses (BTI: series_name, entrances, flat_count, is_emergency, heat/gas/overlap types; Cian: cian_internal_house_id, management_company_id, transport_accessibility_rate, advantages/banks/builders/houses_by_turn jsonb + GIN indexes) - 021: CREATE management_companies + FK to houses.management_company_id - 022: CREATE agents table + agent_id_fk on listings - 023: CREATE offer_price_history (per-listing price change timeline) - 024: CREATE houses_price_dynamics (monthly price/sqm series) - 025: CREATE house_reliability_checks (наш.дом.рф checks) - 026: CREATE external_valuations (unified Avito IMV + Cian Valuation cache) - 027: CREATE EXTENSION pgcrypto + cian_session_cookies (encrypted) - 028: CREATE house_sources, listing_sources, house_address_aliases + address_fingerprint/canonical/merged_into alters Refs: decisions/CianScraper_v1_Implementation_Plan.md Stage 1 decisions/Schema_Cian_SERP_Inventory.md sec 16, 20.3, 26 decisions/Cross_Source_Matching_Strategy.md sec 2 --- .../data/sql/019_listings_alter_cian.sql | 76 +++++++++++++ .../data/sql/020_houses_alter_cian.sql | 69 ++++++++++++ .../data/sql/021_management_companies.sql | 45 ++++++++ .../backend/data/sql/022_agents_table.sql | 38 +++++++ .../data/sql/023_offer_price_history.sql | 30 +++++ .../data/sql/024_houses_price_dynamics.sql | 26 +++++ .../data/sql/025_house_reliability_checks.sql | 29 +++++ .../data/sql/026_external_valuations.sql | 46 ++++++++ .../data/sql/027_cian_session_cookies.sql | 39 +++++++ .../backend/data/sql/028_matching_tables.sql | 103 ++++++++++++++++++ 10 files changed, 501 insertions(+) create mode 100644 tradein-mvp/backend/data/sql/019_listings_alter_cian.sql create mode 100644 tradein-mvp/backend/data/sql/020_houses_alter_cian.sql create mode 100644 tradein-mvp/backend/data/sql/021_management_companies.sql create mode 100644 tradein-mvp/backend/data/sql/022_agents_table.sql create mode 100644 tradein-mvp/backend/data/sql/023_offer_price_history.sql create mode 100644 tradein-mvp/backend/data/sql/024_houses_price_dynamics.sql create mode 100644 tradein-mvp/backend/data/sql/025_house_reliability_checks.sql create mode 100644 tradein-mvp/backend/data/sql/026_external_valuations.sql create mode 100644 tradein-mvp/backend/data/sql/027_cian_session_cookies.sql create mode 100644 tradein-mvp/backend/data/sql/028_matching_tables.sql diff --git a/tradein-mvp/backend/data/sql/019_listings_alter_cian.sql b/tradein-mvp/backend/data/sql/019_listings_alter_cian.sql new file mode 100644 index 00000000..ee9a3945 --- /dev/null +++ b/tradein-mvp/backend/data/sql/019_listings_alter_cian.sql @@ -0,0 +1,76 @@ +-- 019_listings_alter_cian.sql +-- Purpose: Add Cian-specific columns to listings table (SERP + detail fields). +-- Dependencies: listings table created in 002_core_tables.sql / 011_listings_alter.sql +-- Deploy order: Apply after 018_avito_imv_evaluations.sql +-- +-- Fields sourced from: +-- - Schema_Cian_SERP_Inventory sec 16.1 (SERP-level Cian fields) +-- - Schema_Cian_SERP_Inventory sec 20.3 (detail page + stats) +-- - CianScraper_v1_Implementation_Plan Stage 1 + +BEGIN; + +ALTER TABLE listings + -- Cian detail page: window/bathroom/ceiling data + ADD COLUMN IF NOT EXISTS windows_view_type text, -- 'yardAndStreet'/'yard'/'street'/'park' + ADD COLUMN IF NOT EXISTS separate_wcs_count int, -- раздельных санузлов + ADD COLUMN IF NOT EXISTS combined_wcs_count int, -- совмещённых санузлов + ADD COLUMN IF NOT EXISTS ceiling_height numeric(3,2), -- из building.ceilingHeight, напр. 2.5 + + -- Cian detail: repair / finish type (вторичка) + ADD COLUMN IF NOT EXISTS repair_type text, -- without/cosmetic/euro/designer + + -- Cian detail: stats (из offerData.stats) + ADD COLUMN IF NOT EXISTS views_total int, -- суммарные просмотры + ADD COLUMN IF NOT EXISTS views_today int, -- просмотры за сегодня + + -- Cian SERP: living area + bedroom/balcony breakdown + ADD COLUMN IF NOT EXISTS living_area_m2 numeric(7,2), -- жилая площадь (только Cian) + ADD COLUMN IF NOT EXISTS bedrooms_count int, -- кол-во спален + ADD COLUMN IF NOT EXISTS balconies_count int, -- балконов + ADD COLUMN IF NOT EXISTS loggias_count int, -- лоджий + + -- Cian SERP: dedup/matching helpers + ADD COLUMN IF NOT EXISTS description_minhash text, -- Cian LSH minhash for dedup + ADD COLUMN IF NOT EXISTS cadastral_number text, -- кадастровый номер квартиры + + -- Cian SERP: building cadastral (for house matching) + ADD COLUMN IF NOT EXISTS building_cadastral_number text, -- кадастровый номер здания + + -- Cian SERP: seller info (structured, unlike Avito text) + ADD COLUMN IF NOT EXISTS phones jsonb, -- [{countryCode, number, type}, ...] + ADD COLUMN IF NOT EXISTS is_homeowner boolean, -- продаёт собственник (isOwner) + ADD COLUMN IF NOT EXISTS is_pro_seller boolean, -- PRO подписка у продавца + + -- Cian SERP: bargain/sale type flags + ADD COLUMN IF NOT EXISTS bargain_allowed boolean, -- торг возможен + ADD COLUMN IF NOT EXISTS sale_type text, -- 'free'/'alternative'/'new' + + -- Cian SERP: newbuilding house link + ADD COLUMN IF NOT EXISTS house_source text, -- 'cian' (источник house record) + ADD COLUMN IF NOT EXISTS house_ext_id text, -- cian newbuilding id (строка) + + -- Cian SERP: metro stations near offer + ADD COLUMN IF NOT EXISTS metro_stations jsonb; -- [{name, distanceKm, travelTime, travelType}, ...] + +-- Partial index for fast minhash dedup lookups +CREATE INDEX IF NOT EXISTS listings_minhash_idx + ON listings (description_minhash) + WHERE description_minhash IS NOT NULL; + +-- Partial index on cadastral_number for Tier-0 apartment matching +CREATE INDEX IF NOT EXISTS listings_cadastral_idx + ON listings (cadastral_number) + WHERE cadastral_number IS NOT NULL; + +-- Partial index on building_cadastral_number for house matching +CREATE INDEX IF NOT EXISTS listings_bld_cadastral_idx + ON listings (building_cadastral_number) + WHERE building_cadastral_number IS NOT NULL; + +-- GIN on metro_stations for JSON querying +CREATE INDEX IF NOT EXISTS listings_metro_stations_gin_idx + ON listings USING GIN (metro_stations) + WHERE metro_stations IS NOT NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/020_houses_alter_cian.sql b/tradein-mvp/backend/data/sql/020_houses_alter_cian.sql new file mode 100644 index 00000000..56be6a0f --- /dev/null +++ b/tradein-mvp/backend/data/sql/020_houses_alter_cian.sql @@ -0,0 +1,69 @@ +-- 020_houses_alter_cian.sql +-- Purpose: Add Cian-specific and BTI columns to houses table. +-- Dependencies: +-- - houses table: 009_houses.sql +-- - management_companies table: 021_management_companies.sql +-- NOTE: management_company_id FK added here but the REFERENCES target (management_companies) +-- must exist first. Apply 021 BEFORE adding the FK column, OR apply 020 without the FK +-- and add FK separately. To keep idempotent ordering: management_companies is created in 021, +-- so 021 must be applied before 020 in production. Alternatively, add FK in 021 via ALTER. +-- DECISION: management_company_id column added without FK here; FK constraint added in 021 +-- after management_companies table is created. This file only adds plain bigint column. +-- Deploy order: Apply after 019. Apply 021 after 020 to complete FK wiring. +-- +-- Sources: Schema_Cian_SERP_Inventory sec 16.5, 20.3, 26.3 + +BEGIN; + +ALTER TABLE houses + -- BTI data (from offerData.bti.houseData — вторичка detail page) + ADD COLUMN IF NOT EXISTS series_name text, -- 'I-447' / 'II-49' / etc (Cian BTI) + ADD COLUMN IF NOT EXISTS entrances int, -- подъездов в доме + ADD COLUMN IF NOT EXISTS flat_count int, -- квартир в доме (liquidity proxy) + ADD COLUMN IF NOT EXISTS is_emergency boolean, -- аварийный дом (red flag!) + ADD COLUMN IF NOT EXISTS heat_supply_type text, -- central/individual/none + ADD COLUMN IF NOT EXISTS gas_supply_type text, -- central/none + ADD COLUMN IF NOT EXISTS overlap_type text, -- concrete/wood/steel + + -- Cian newbuilding / valuation specific + ADD COLUMN IF NOT EXISTS cian_internal_house_id bigint, -- Cian's houseId (filters.houseId) + ADD COLUMN IF NOT EXISTS management_company_id bigint, -- FK to management_companies(id) added in 021 + ADD COLUMN IF NOT EXISTS transport_accessibility_rate int, -- 0-10 transport score + + -- Cian newbuilding jsonb arrays + ADD COLUMN IF NOT EXISTS advantages jsonb, -- typed advantage list + ADD COLUMN IF NOT EXISTS banks jsonb, -- банки-партнёры (ipoteka) + ADD COLUMN IF NOT EXISTS builders jsonb, -- застройщики (может быть несколько) + ADD COLUMN IF NOT EXISTS houses_by_turn jsonb; -- группировка корпусов по очередям + +-- Partial index on series_name for BTI series analytics +CREATE INDEX IF NOT EXISTS houses_series_idx + ON houses (series_name) + WHERE series_name IS NOT NULL; + +-- Partial index for emergency flag (rare, useful for risk filter) +CREATE INDEX IF NOT EXISTS houses_emergency_idx + ON houses (is_emergency) + WHERE is_emergency = true; + +-- Partial index for Cian internal house ID (cross-source matching) +CREATE INDEX IF NOT EXISTS houses_cian_iid_idx + ON houses (cian_internal_house_id) + WHERE cian_internal_house_id IS NOT NULL; + +-- GIN on advantages for feature-based filtering +CREATE INDEX IF NOT EXISTS houses_advantages_gin_idx + ON houses USING GIN (advantages) + WHERE advantages IS NOT NULL; + +-- GIN on banks +CREATE INDEX IF NOT EXISTS houses_banks_gin_idx + ON houses USING GIN (banks) + WHERE banks IS NOT NULL; + +-- GIN on builders +CREATE INDEX IF NOT EXISTS houses_builders_gin_idx + ON houses USING GIN (builders) + WHERE builders IS NOT NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/021_management_companies.sql b/tradein-mvp/backend/data/sql/021_management_companies.sql new file mode 100644 index 00000000..05915c21 --- /dev/null +++ b/tradein-mvp/backend/data/sql/021_management_companies.sql @@ -0,0 +1,45 @@ +-- 021_management_companies.sql +-- Purpose: Create management_companies table for ТСЖ/УК data from Cian Valuation Calculator. +-- Also adds FK constraint to houses.management_company_id (column added in 020). +-- Dependencies: +-- - 020_houses_alter_cian.sql must be applied first (adds management_company_id column to houses) +-- Deploy order: Apply after 020. +-- +-- Sources: Schema_Cian_SERP_Inventory sec 26.3 + +BEGIN; + +CREATE TABLE IF NOT EXISTS management_companies ( + id bigserial PRIMARY KEY, + name text NOT NULL, -- 'ТСЖ "УЧИТЕЛЕЙ, 18"' + phones text[], -- array of phone strings + email text, + opening_hours text, -- raw string or serialised schedule + chief_name text, -- ФИО руководителя + ext_source text NOT NULL DEFAULT 'cian', + ext_id bigint, -- Cian's internal management company ID if available + created_at timestamptz NOT NULL DEFAULT NOW(), + UNIQUE (ext_source, name, ext_id) +); + +-- ext_source + ext_id for dedup when Cian provides numeric ID +CREATE INDEX IF NOT EXISTS mc_ext_source_id_idx + ON management_companies (ext_source, ext_id) + WHERE ext_id IS NOT NULL; + +-- Text search on company name +CREATE INDEX IF NOT EXISTS mc_name_idx + ON management_companies (name); + +-- Add FK from houses.management_company_id to this table +-- Column management_company_id was added in 020_houses_alter_cian.sql as plain bigint +ALTER TABLE houses + DROP CONSTRAINT IF EXISTS houses_management_company_id_fkey; + +ALTER TABLE houses + ADD CONSTRAINT houses_management_company_id_fkey + FOREIGN KEY (management_company_id) + REFERENCES management_companies (id) + ON DELETE SET NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/022_agents_table.sql b/tradein-mvp/backend/data/sql/022_agents_table.sql new file mode 100644 index 00000000..78027613 --- /dev/null +++ b/tradein-mvp/backend/data/sql/022_agents_table.sql @@ -0,0 +1,38 @@ +-- 022_agents_table.sql +-- Purpose: Create agents table for Cian seller/agent profiles; link to listings. +-- Dependencies: +-- - listings table (002_core_tables.sql / 011_listings_alter.sql) +-- Deploy order: Apply after 021. +-- +-- Sources: Schema_Cian_SERP_Inventory sec 20.3 (offerData.agent structure) + +BEGIN; + +CREATE TABLE IF NOT EXISTS agents ( + id bigserial PRIMARY KEY, + ext_source text NOT NULL, -- 'cian' / 'avito' + ext_agent_id text NOT NULL, -- Cian cianUserId (as text for flexibility) + company_name text, -- 'Новосёл' (агентство) + is_pro boolean, -- PRO подписка + skills jsonb, -- [{id, name, parentName}, ...] + account_type text, -- specialist/agent/agency + created_at timestamptz NOT NULL DEFAULT NOW(), + UNIQUE (ext_source, ext_agent_id) +); + +CREATE INDEX IF NOT EXISTS agents_source_ext_idx + ON agents (ext_source, ext_agent_id); + +CREATE INDEX IF NOT EXISTS agents_company_idx + ON agents (company_name) + WHERE company_name IS NOT NULL; + +-- Link listings to agent profiles +ALTER TABLE listings + ADD COLUMN IF NOT EXISTS agent_id_fk bigint REFERENCES agents(id) ON DELETE SET NULL; + +CREATE INDEX IF NOT EXISTS listings_agent_idx + ON listings (agent_id_fk) + WHERE agent_id_fk IS NOT NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/023_offer_price_history.sql b/tradein-mvp/backend/data/sql/023_offer_price_history.sql new file mode 100644 index 00000000..56532b77 --- /dev/null +++ b/tradein-mvp/backend/data/sql/023_offer_price_history.sql @@ -0,0 +1,30 @@ +-- 023_offer_price_history.sql +-- Purpose: Per-listing price change history from Cian offerData.priceChanges +-- (and optionally Avito equivalent when available). +-- Dependencies: +-- - listings table (002_core_tables.sql) +-- Deploy order: Apply after 022. +-- +-- Sources: Schema_Cian_SERP_Inventory sec 16.2, 20.2 (priceChanges array) + +BEGIN; + +CREATE TABLE IF NOT EXISTS offer_price_history ( + id bigserial PRIMARY KEY, + listing_id bigint NOT NULL REFERENCES listings(id) ON DELETE CASCADE, + change_time timestamptz NOT NULL, -- changeTime from Cian (ISO-8601) + price_rub numeric NOT NULL, -- priceData.price в рублях + diff_percent numeric(5,2), -- вычисленный % относительно предыдущей точки + source text, -- 'cian' / 'avito' + recorded_at timestamptz NOT NULL DEFAULT NOW() +); + +-- Primary access pattern: get ordered history for a listing +CREATE INDEX IF NOT EXISTS oph_listing_time_idx + ON offer_price_history (listing_id, change_time DESC); + +-- Source-level stats +CREATE INDEX IF NOT EXISTS oph_source_time_idx + ON offer_price_history (source, change_time DESC); + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/024_houses_price_dynamics.sql b/tradein-mvp/backend/data/sql/024_houses_price_dynamics.sql new file mode 100644 index 00000000..24e5c439 --- /dev/null +++ b/tradein-mvp/backend/data/sql/024_houses_price_dynamics.sql @@ -0,0 +1,26 @@ +-- 024_houses_price_dynamics.sql +-- Purpose: Monthly price time-series per house (from Cian newbuilding realtyValuation.priceDynamics). +-- Stores per room_count / prices_type / period granularity as provided by Cian. +-- Dependencies: +-- - houses table (009_houses.sql) +-- Deploy order: Apply after 023. +-- +-- Sources: Schema_Cian_SERP_Inventory sec 16.3 + +BEGIN; + +CREATE TABLE IF NOT EXISTS houses_price_dynamics ( + id bigserial PRIMARY KEY, + house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE, + month_date date NOT NULL, -- '2025-11-01' (first of month) + price_per_sqm numeric NOT NULL, -- avg price per sqm for that month + source text NOT NULL DEFAULT 'cian_realty_valuation', + recorded_at timestamptz NOT NULL DEFAULT NOW(), + UNIQUE (house_id, month_date, source) +); + +-- Primary read pattern: time-ordered series for a house +CREATE INDEX IF NOT EXISTS hpd_house_date_idx + ON houses_price_dynamics (house_id, month_date DESC); + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/025_house_reliability_checks.sql b/tradein-mvp/backend/data/sql/025_house_reliability_checks.sql new file mode 100644 index 00000000..5cd24e1c --- /dev/null +++ b/tradein-mvp/backend/data/sql/025_house_reliability_checks.sql @@ -0,0 +1,29 @@ +-- 025_house_reliability_checks.sql +-- Purpose: наш.дом.рф reliability check results scraped from Cian newbuilding pages. +-- Stores per-house check status and detail array. +-- Dependencies: +-- - houses table (009_houses.sql) +-- Deploy order: Apply after 024. +-- +-- Sources: Schema_Cian_SERP_Inventory sec 16.4 + +BEGIN; + +CREATE TABLE IF NOT EXISTS house_reliability_checks ( + id bigserial PRIMARY KEY, + house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE, + check_status text, -- 'reliable'/'problematic'/'unchecked' + check_name text, -- human-readable check name / title + details jsonb, -- [{type, title, iconType, ...}] raw checks array + source text NOT NULL DEFAULT 'cian_nashdom', + recorded_at timestamptz NOT NULL DEFAULT NOW() +); + +CREATE INDEX IF NOT EXISTS hrc_house_idx + ON house_reliability_checks (house_id); + +CREATE INDEX IF NOT EXISTS hrc_status_idx + ON house_reliability_checks (check_status) + WHERE check_status IS NOT NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/026_external_valuations.sql b/tradein-mvp/backend/data/sql/026_external_valuations.sql new file mode 100644 index 00000000..ce887665 --- /dev/null +++ b/tradein-mvp/backend/data/sql/026_external_valuations.sql @@ -0,0 +1,46 @@ +-- 026_external_valuations.sql +-- Purpose: Unified cache for external valuation APIs — Avito IMV and Cian Valuation Calculator. +-- Replaces/supersedes avito_imv_evaluations (018) for multi-source use. +-- 018 table kept for backward compat; new code writes here. +-- Dependencies: +-- - None (standalone cache table) +-- Deploy order: Apply after 025. +-- +-- Sources: Schema_Cian_SERP_Inventory sec 26.3 (external_valuations DDL) +-- CianScraper_v1_Implementation_Plan Stage 1 + +BEGIN; + +CREATE TABLE IF NOT EXISTS external_valuations ( + id bigserial PRIMARY KEY, + source text NOT NULL, -- 'avito_imv' / 'cian_valuation' + cache_key text NOT NULL, -- sha256(address+area+rooms+floor+repair+deal) + -- Input snapshot + address text, + total_area numeric(8,2), + rooms_count int, + floor int, + total_floors int, + repair_type text, -- without/cosmetic/euro/design + deal_type text, -- sale/rent + -- Output + price_rub numeric, -- primary estimate price + accuracy numeric, -- accuracy % (Cian-specific; 0-100) + raw_payload jsonb, -- full API response + chart jsonb, -- [{date, price}, ...] monthly points + -- Cache TTL + fetched_at timestamptz NOT NULL DEFAULT NOW(), + expires_at timestamptz NOT NULL, -- fetched_at + 24h typically + UNIQUE (source, cache_key) +); + +-- Cache lookup: valid entries only +CREATE INDEX IF NOT EXISTS ev_source_expires_idx + ON external_valuations (source, expires_at DESC); + +-- Address-based lookup (for analytics / dedup) +CREATE INDEX IF NOT EXISTS ev_address_idx + ON external_valuations (address) + WHERE address IS NOT NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/027_cian_session_cookies.sql b/tradein-mvp/backend/data/sql/027_cian_session_cookies.sql new file mode 100644 index 00000000..ecd771bf --- /dev/null +++ b/tradein-mvp/backend/data/sql/027_cian_session_cookies.sql @@ -0,0 +1,39 @@ +-- 027_cian_session_cookies.sql +-- Purpose: Encrypted storage for Cian browser session cookies needed to call +-- the Valuation Calculator endpoint (auth required). +-- Uses pgcrypto pgp_sym_encrypt for AES encryption at rest. +-- Dependencies: +-- - pgcrypto extension (installed here via CREATE EXTENSION IF NOT EXISTS) +-- Deploy order: Apply after 026. +-- +-- Security notes: +-- - cookies_encrypted stores AES-encrypted JSON blob via pgp_sym_encrypt. +-- - Encryption key lives in .env.runtime (COOKIE_ENCRYPTION_KEY), never in DB. +-- - Access restricted to admin-token-gated API endpoint only. +-- +-- Sources: CianScraper_v1_Implementation_Plan (Cookie auth section) + +BEGIN; + +CREATE EXTENSION IF NOT EXISTS pgcrypto; + +CREATE TABLE IF NOT EXISTS cian_session_cookies ( + account_user_id bigint PRIMARY KEY, -- Cian user_id extracted from state.user.userId + cookies_encrypted bytea NOT NULL, -- pgp_sym_encrypt(json_cookies, key) + expires_at_estimate timestamptz NOT NULL, -- estimated expiry (~30 days from upload) + uploaded_at timestamptz NOT NULL DEFAULT NOW(), + last_used_at timestamptz, -- set on each successful valuation call + last_invalid_at timestamptz, -- set when isAuthenticated=false detected + notes text -- e.g. 'Account: lekss361@cian.ru' +); + +-- Most recent upload first (admin dashboard ordering) +CREATE INDEX IF NOT EXISTS cian_cookies_uploaded_idx + ON cian_session_cookies (uploaded_at DESC); + +-- Active session lookup (non-expired and not invalidated) +CREATE INDEX IF NOT EXISTS cian_cookies_active_idx + ON cian_session_cookies (expires_at_estimate) + WHERE last_invalid_at IS NULL; + +COMMIT; diff --git a/tradein-mvp/backend/data/sql/028_matching_tables.sql b/tradein-mvp/backend/data/sql/028_matching_tables.sql new file mode 100644 index 00000000..cca491cd --- /dev/null +++ b/tradein-mvp/backend/data/sql/028_matching_tables.sql @@ -0,0 +1,103 @@ +-- 028_matching_tables.sql +-- Purpose: Cross-source matching link tables for deduplication and enrichment. +-- Implements 3-tier house + apartment matching per Cross_Source_Matching_Strategy. +-- house_sources — one canonical house ↔ many external (source, ext_id) pairs +-- listing_sources — one canonical listing ↔ many external (source, ext_id) pairs +-- house_address_aliases — manual overrides for corpus/address format mismatches +-- Dependencies: +-- - houses table (009_houses.sql) +-- - listings table (002_core_tables.sql) +-- Deploy order: Apply after 027. +-- +-- Sources: Cross_Source_Matching_Strategy sec 2 + +BEGIN; + +-- address_fingerprint column on houses (needed for Tier-1 matching algorithm) +ALTER TABLE houses + ADD COLUMN IF NOT EXISTS address_fingerprint text; + +CREATE INDEX IF NOT EXISTS houses_addr_fp_idx + ON houses (address_fingerprint) + WHERE address_fingerprint IS NOT NULL; + +-- canonical / merged_into fields on listings (needed for dedup workflow) +ALTER TABLE listings + ADD COLUMN IF NOT EXISTS canonical boolean DEFAULT true, + ADD COLUMN IF NOT EXISTS merged_into bigint REFERENCES listings(id) ON DELETE SET NULL; + +CREATE INDEX IF NOT EXISTS listings_merged_into_idx + ON listings (merged_into) + WHERE merged_into IS NOT NULL; + +-- ----------------------------------------------------------------------- +-- house_sources: link canonical house to external source records +-- ----------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS house_sources ( + id bigserial PRIMARY KEY, + house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE, + ext_source text NOT NULL, -- 'avito' / 'cian' / 'yandex' / 'n1' + ext_id text NOT NULL, -- source-specific house/newbuilding ID + confidence real NOT NULL, -- 0.0–1.0 + matched_method text NOT NULL, -- 'cadastr_exact'/'address_geo_year'/'geo_only'/'new'/'manual' + matched_at timestamptz NOT NULL DEFAULT NOW(), + UNIQUE (ext_source, ext_id) +); + +CREATE INDEX IF NOT EXISTS house_sources_house_idx + ON house_sources (house_id); + +-- Low-confidence matches for review queue +CREATE INDEX IF NOT EXISTS house_sources_low_conf_idx + ON house_sources (confidence) + WHERE confidence < 0.85; + +-- ----------------------------------------------------------------------- +-- listing_sources: link canonical listing to external source records +-- ----------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS listing_sources ( + id bigserial PRIMARY KEY, + listing_id bigint NOT NULL REFERENCES listings(id) ON DELETE CASCADE, + ext_source text NOT NULL, -- 'avito' / 'cian' + ext_id text NOT NULL, -- Avito source_id / Cian cianId (as text) + confidence real NOT NULL, -- 0.0–1.0 + matched_method text NOT NULL, -- 'cadastr_exact'/'minhash'/'composite'/'new'/'manual' + matched_at timestamptz NOT NULL DEFAULT NOW(), + source_url text, -- URL объявления на источнике + last_scraped_at timestamptz, -- последний раз видели на источнике + UNIQUE (ext_source, ext_id) +); + +CREATE INDEX IF NOT EXISTS listing_sources_listing_idx + ON listing_sources (listing_id); + +-- Low-confidence listing matches for review +CREATE INDEX IF NOT EXISTS listing_sources_low_conf_idx + ON listing_sources (confidence) + WHERE confidence < 0.85; + +-- Recency index for stale-data detection +CREATE INDEX IF NOT EXISTS listing_sources_scraped_idx + ON listing_sources (ext_source, last_scraped_at DESC); + +-- ----------------------------------------------------------------------- +-- house_address_aliases: manual mapping for corpus/address format mismatches +-- e.g. 'ул. Постовского 17А' ↔ 'ул. Постовского 17 корп.1' +-- ----------------------------------------------------------------------- +CREATE TABLE IF NOT EXISTS house_address_aliases ( + id bigserial PRIMARY KEY, + house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE, + normalized_address text NOT NULL, -- normalized form (output of normalize_address()) + fingerprint text, -- SHA1 of normalized_address + source text, -- which scraper/dataset provided this alias + UNIQUE (normalized_address) +); + +CREATE INDEX IF NOT EXISTS haa_house_idx + ON house_address_aliases (house_id); + +CREATE INDEX IF NOT EXISTS haa_fingerprint_idx + ON house_address_aliases (fingerprint) + WHERE fingerprint IS NOT NULL; + +COMMIT;