feat(tradein): SQL migrations 019-028 — Cian schema + matching + cookies (#445)
Stage 1 / Wave 1 of CianScraper v1 — schema foundation for 4 Cian sources (SERP/detail/newbuilding/valuation) + cross-source matching. 10 SQL migrations: - 019-020: ALTER listings/houses with Cian-specific cols - 021-022: management_companies, agents tables - 023-024: offer_price_history, houses_price_dynamics - 025-026: house_reliability_checks, external_valuations cache - 027: cian_session_cookies (pgcrypto encryption) - 028: matching tables (house_sources/listing_sources/house_address_aliases) All migrations are idempotent (BEGIN/COMMIT + IF NOT EXISTS). Reviewed by deep-code-reviewer.
This commit is contained in:
commit
23e6cbc773
10 changed files with 501 additions and 0 deletions
76
tradein-mvp/backend/data/sql/019_listings_alter_cian.sql
Normal file
76
tradein-mvp/backend/data/sql/019_listings_alter_cian.sql
Normal file
|
|
@ -0,0 +1,76 @@
|
|||
-- 019_listings_alter_cian.sql
|
||||
-- Purpose: Add Cian-specific columns to listings table (SERP + detail fields).
|
||||
-- Dependencies: listings table created in 002_core_tables.sql / 011_listings_alter.sql
|
||||
-- Deploy order: Apply after 018_avito_imv_evaluations.sql
|
||||
--
|
||||
-- Fields sourced from:
|
||||
-- - Schema_Cian_SERP_Inventory sec 16.1 (SERP-level Cian fields)
|
||||
-- - Schema_Cian_SERP_Inventory sec 20.3 (detail page + stats)
|
||||
-- - CianScraper_v1_Implementation_Plan Stage 1
|
||||
|
||||
BEGIN;
|
||||
|
||||
ALTER TABLE listings
|
||||
-- Cian detail page: window/bathroom/ceiling data
|
||||
ADD COLUMN IF NOT EXISTS windows_view_type text, -- 'yardAndStreet'/'yard'/'street'/'park'
|
||||
ADD COLUMN IF NOT EXISTS separate_wcs_count int, -- раздельных санузлов
|
||||
ADD COLUMN IF NOT EXISTS combined_wcs_count int, -- совмещённых санузлов
|
||||
ADD COLUMN IF NOT EXISTS ceiling_height numeric(3,2), -- из building.ceilingHeight, напр. 2.5
|
||||
|
||||
-- Cian detail: repair / finish type (вторичка)
|
||||
ADD COLUMN IF NOT EXISTS repair_type text, -- without/cosmetic/euro/designer
|
||||
|
||||
-- Cian detail: stats (из offerData.stats)
|
||||
ADD COLUMN IF NOT EXISTS views_total int, -- суммарные просмотры
|
||||
ADD COLUMN IF NOT EXISTS views_today int, -- просмотры за сегодня
|
||||
|
||||
-- Cian SERP: living area + bedroom/balcony breakdown
|
||||
ADD COLUMN IF NOT EXISTS living_area_m2 numeric(7,2), -- жилая площадь (только Cian)
|
||||
ADD COLUMN IF NOT EXISTS bedrooms_count int, -- кол-во спален
|
||||
ADD COLUMN IF NOT EXISTS balconies_count int, -- балконов
|
||||
ADD COLUMN IF NOT EXISTS loggias_count int, -- лоджий
|
||||
|
||||
-- Cian SERP: dedup/matching helpers
|
||||
ADD COLUMN IF NOT EXISTS description_minhash text, -- Cian LSH minhash for dedup
|
||||
ADD COLUMN IF NOT EXISTS cadastral_number text, -- кадастровый номер квартиры
|
||||
|
||||
-- Cian SERP: building cadastral (for house matching)
|
||||
ADD COLUMN IF NOT EXISTS building_cadastral_number text, -- кадастровый номер здания
|
||||
|
||||
-- Cian SERP: seller info (structured, unlike Avito text)
|
||||
ADD COLUMN IF NOT EXISTS phones jsonb, -- [{countryCode, number, type}, ...]
|
||||
ADD COLUMN IF NOT EXISTS is_homeowner boolean, -- продаёт собственник (isOwner)
|
||||
ADD COLUMN IF NOT EXISTS is_pro_seller boolean, -- PRO подписка у продавца
|
||||
|
||||
-- Cian SERP: bargain/sale type flags
|
||||
ADD COLUMN IF NOT EXISTS bargain_allowed boolean, -- торг возможен
|
||||
ADD COLUMN IF NOT EXISTS sale_type text, -- 'free'/'alternative'/'new'
|
||||
|
||||
-- Cian SERP: newbuilding house link
|
||||
ADD COLUMN IF NOT EXISTS house_source text, -- 'cian' (источник house record)
|
||||
ADD COLUMN IF NOT EXISTS house_ext_id text, -- cian newbuilding id (строка)
|
||||
|
||||
-- Cian SERP: metro stations near offer
|
||||
ADD COLUMN IF NOT EXISTS metro_stations jsonb; -- [{name, distanceKm, travelTime, travelType}, ...]
|
||||
|
||||
-- Partial index for fast minhash dedup lookups
|
||||
CREATE INDEX IF NOT EXISTS listings_minhash_idx
|
||||
ON listings (description_minhash)
|
||||
WHERE description_minhash IS NOT NULL;
|
||||
|
||||
-- Partial index on cadastral_number for Tier-0 apartment matching
|
||||
CREATE INDEX IF NOT EXISTS listings_cadastral_idx
|
||||
ON listings (cadastral_number)
|
||||
WHERE cadastral_number IS NOT NULL;
|
||||
|
||||
-- Partial index on building_cadastral_number for house matching
|
||||
CREATE INDEX IF NOT EXISTS listings_bld_cadastral_idx
|
||||
ON listings (building_cadastral_number)
|
||||
WHERE building_cadastral_number IS NOT NULL;
|
||||
|
||||
-- GIN on metro_stations for JSON querying
|
||||
CREATE INDEX IF NOT EXISTS listings_metro_stations_gin_idx
|
||||
ON listings USING GIN (metro_stations)
|
||||
WHERE metro_stations IS NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
69
tradein-mvp/backend/data/sql/020_houses_alter_cian.sql
Normal file
69
tradein-mvp/backend/data/sql/020_houses_alter_cian.sql
Normal file
|
|
@ -0,0 +1,69 @@
|
|||
-- 020_houses_alter_cian.sql
|
||||
-- Purpose: Add Cian-specific and BTI columns to houses table.
|
||||
-- Dependencies:
|
||||
-- - houses table: 009_houses.sql
|
||||
-- - management_companies table: 021_management_companies.sql
|
||||
-- NOTE: management_company_id FK added here but the REFERENCES target (management_companies)
|
||||
-- must exist first. Apply 021 BEFORE adding the FK column, OR apply 020 without the FK
|
||||
-- and add FK separately. To keep idempotent ordering: management_companies is created in 021,
|
||||
-- so 021 must be applied before 020 in production. Alternatively, add FK in 021 via ALTER.
|
||||
-- DECISION: management_company_id column added without FK here; FK constraint added in 021
|
||||
-- after management_companies table is created. This file only adds plain bigint column.
|
||||
-- Deploy order: Apply after 019. Apply 021 after 020 to complete FK wiring.
|
||||
--
|
||||
-- Sources: Schema_Cian_SERP_Inventory sec 16.5, 20.3, 26.3
|
||||
|
||||
BEGIN;
|
||||
|
||||
ALTER TABLE houses
|
||||
-- BTI data (from offerData.bti.houseData — вторичка detail page)
|
||||
ADD COLUMN IF NOT EXISTS series_name text, -- 'I-447' / 'II-49' / etc (Cian BTI)
|
||||
ADD COLUMN IF NOT EXISTS entrances int, -- подъездов в доме
|
||||
ADD COLUMN IF NOT EXISTS flat_count int, -- квартир в доме (liquidity proxy)
|
||||
ADD COLUMN IF NOT EXISTS is_emergency boolean, -- аварийный дом (red flag!)
|
||||
ADD COLUMN IF NOT EXISTS heat_supply_type text, -- central/individual/none
|
||||
ADD COLUMN IF NOT EXISTS gas_supply_type text, -- central/none
|
||||
ADD COLUMN IF NOT EXISTS overlap_type text, -- concrete/wood/steel
|
||||
|
||||
-- Cian newbuilding / valuation specific
|
||||
ADD COLUMN IF NOT EXISTS cian_internal_house_id bigint, -- Cian's houseId (filters.houseId)
|
||||
ADD COLUMN IF NOT EXISTS management_company_id bigint, -- FK to management_companies(id) added in 021
|
||||
ADD COLUMN IF NOT EXISTS transport_accessibility_rate int, -- 0-10 transport score
|
||||
|
||||
-- Cian newbuilding jsonb arrays
|
||||
ADD COLUMN IF NOT EXISTS advantages jsonb, -- typed advantage list
|
||||
ADD COLUMN IF NOT EXISTS banks jsonb, -- банки-партнёры (ipoteka)
|
||||
ADD COLUMN IF NOT EXISTS builders jsonb, -- застройщики (может быть несколько)
|
||||
ADD COLUMN IF NOT EXISTS houses_by_turn jsonb; -- группировка корпусов по очередям
|
||||
|
||||
-- Partial index on series_name for BTI series analytics
|
||||
CREATE INDEX IF NOT EXISTS houses_series_idx
|
||||
ON houses (series_name)
|
||||
WHERE series_name IS NOT NULL;
|
||||
|
||||
-- Partial index for emergency flag (rare, useful for risk filter)
|
||||
CREATE INDEX IF NOT EXISTS houses_emergency_idx
|
||||
ON houses (is_emergency)
|
||||
WHERE is_emergency = true;
|
||||
|
||||
-- Partial index for Cian internal house ID (cross-source matching)
|
||||
CREATE INDEX IF NOT EXISTS houses_cian_iid_idx
|
||||
ON houses (cian_internal_house_id)
|
||||
WHERE cian_internal_house_id IS NOT NULL;
|
||||
|
||||
-- GIN on advantages for feature-based filtering
|
||||
CREATE INDEX IF NOT EXISTS houses_advantages_gin_idx
|
||||
ON houses USING GIN (advantages)
|
||||
WHERE advantages IS NOT NULL;
|
||||
|
||||
-- GIN on banks
|
||||
CREATE INDEX IF NOT EXISTS houses_banks_gin_idx
|
||||
ON houses USING GIN (banks)
|
||||
WHERE banks IS NOT NULL;
|
||||
|
||||
-- GIN on builders
|
||||
CREATE INDEX IF NOT EXISTS houses_builders_gin_idx
|
||||
ON houses USING GIN (builders)
|
||||
WHERE builders IS NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
45
tradein-mvp/backend/data/sql/021_management_companies.sql
Normal file
45
tradein-mvp/backend/data/sql/021_management_companies.sql
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
-- 021_management_companies.sql
|
||||
-- Purpose: Create management_companies table for ТСЖ/УК data from Cian Valuation Calculator.
|
||||
-- Also adds FK constraint to houses.management_company_id (column added in 020).
|
||||
-- Dependencies:
|
||||
-- - 020_houses_alter_cian.sql must be applied first (adds management_company_id column to houses)
|
||||
-- Deploy order: Apply after 020.
|
||||
--
|
||||
-- Sources: Schema_Cian_SERP_Inventory sec 26.3
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS management_companies (
|
||||
id bigserial PRIMARY KEY,
|
||||
name text NOT NULL, -- 'ТСЖ "УЧИТЕЛЕЙ, 18"'
|
||||
phones text[], -- array of phone strings
|
||||
email text,
|
||||
opening_hours text, -- raw string or serialised schedule
|
||||
chief_name text, -- ФИО руководителя
|
||||
ext_source text NOT NULL DEFAULT 'cian',
|
||||
ext_id bigint, -- Cian's internal management company ID if available
|
||||
created_at timestamptz NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (ext_source, name, ext_id)
|
||||
);
|
||||
|
||||
-- ext_source + ext_id for dedup when Cian provides numeric ID
|
||||
CREATE INDEX IF NOT EXISTS mc_ext_source_id_idx
|
||||
ON management_companies (ext_source, ext_id)
|
||||
WHERE ext_id IS NOT NULL;
|
||||
|
||||
-- Text search on company name
|
||||
CREATE INDEX IF NOT EXISTS mc_name_idx
|
||||
ON management_companies (name);
|
||||
|
||||
-- Add FK from houses.management_company_id to this table
|
||||
-- Column management_company_id was added in 020_houses_alter_cian.sql as plain bigint
|
||||
ALTER TABLE houses
|
||||
DROP CONSTRAINT IF EXISTS houses_management_company_id_fkey;
|
||||
|
||||
ALTER TABLE houses
|
||||
ADD CONSTRAINT houses_management_company_id_fkey
|
||||
FOREIGN KEY (management_company_id)
|
||||
REFERENCES management_companies (id)
|
||||
ON DELETE SET NULL;
|
||||
|
||||
COMMIT;
|
||||
38
tradein-mvp/backend/data/sql/022_agents_table.sql
Normal file
38
tradein-mvp/backend/data/sql/022_agents_table.sql
Normal file
|
|
@ -0,0 +1,38 @@
|
|||
-- 022_agents_table.sql
|
||||
-- Purpose: Create agents table for Cian seller/agent profiles; link to listings.
|
||||
-- Dependencies:
|
||||
-- - listings table (002_core_tables.sql / 011_listings_alter.sql)
|
||||
-- Deploy order: Apply after 021.
|
||||
--
|
||||
-- Sources: Schema_Cian_SERP_Inventory sec 20.3 (offerData.agent structure)
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS agents (
|
||||
id bigserial PRIMARY KEY,
|
||||
ext_source text NOT NULL, -- 'cian' / 'avito'
|
||||
ext_agent_id text NOT NULL, -- Cian cianUserId (as text for flexibility)
|
||||
company_name text, -- 'Новосёл' (агентство)
|
||||
is_pro boolean, -- PRO подписка
|
||||
skills jsonb, -- [{id, name, parentName}, ...]
|
||||
account_type text, -- specialist/agent/agency
|
||||
created_at timestamptz NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (ext_source, ext_agent_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS agents_source_ext_idx
|
||||
ON agents (ext_source, ext_agent_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS agents_company_idx
|
||||
ON agents (company_name)
|
||||
WHERE company_name IS NOT NULL;
|
||||
|
||||
-- Link listings to agent profiles
|
||||
ALTER TABLE listings
|
||||
ADD COLUMN IF NOT EXISTS agent_id_fk bigint REFERENCES agents(id) ON DELETE SET NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS listings_agent_idx
|
||||
ON listings (agent_id_fk)
|
||||
WHERE agent_id_fk IS NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
30
tradein-mvp/backend/data/sql/023_offer_price_history.sql
Normal file
30
tradein-mvp/backend/data/sql/023_offer_price_history.sql
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
-- 023_offer_price_history.sql
|
||||
-- Purpose: Per-listing price change history from Cian offerData.priceChanges
|
||||
-- (and optionally Avito equivalent when available).
|
||||
-- Dependencies:
|
||||
-- - listings table (002_core_tables.sql)
|
||||
-- Deploy order: Apply after 022.
|
||||
--
|
||||
-- Sources: Schema_Cian_SERP_Inventory sec 16.2, 20.2 (priceChanges array)
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS offer_price_history (
|
||||
id bigserial PRIMARY KEY,
|
||||
listing_id bigint NOT NULL REFERENCES listings(id) ON DELETE CASCADE,
|
||||
change_time timestamptz NOT NULL, -- changeTime from Cian (ISO-8601)
|
||||
price_rub numeric NOT NULL, -- priceData.price в рублях
|
||||
diff_percent numeric(5,2), -- вычисленный % относительно предыдущей точки
|
||||
source text, -- 'cian' / 'avito'
|
||||
recorded_at timestamptz NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
-- Primary access pattern: get ordered history for a listing
|
||||
CREATE INDEX IF NOT EXISTS oph_listing_time_idx
|
||||
ON offer_price_history (listing_id, change_time DESC);
|
||||
|
||||
-- Source-level stats
|
||||
CREATE INDEX IF NOT EXISTS oph_source_time_idx
|
||||
ON offer_price_history (source, change_time DESC);
|
||||
|
||||
COMMIT;
|
||||
26
tradein-mvp/backend/data/sql/024_houses_price_dynamics.sql
Normal file
26
tradein-mvp/backend/data/sql/024_houses_price_dynamics.sql
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
-- 024_houses_price_dynamics.sql
|
||||
-- Purpose: Monthly price time-series per house (from Cian newbuilding realtyValuation.priceDynamics).
|
||||
-- Stores per room_count / prices_type / period granularity as provided by Cian.
|
||||
-- Dependencies:
|
||||
-- - houses table (009_houses.sql)
|
||||
-- Deploy order: Apply after 023.
|
||||
--
|
||||
-- Sources: Schema_Cian_SERP_Inventory sec 16.3
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS houses_price_dynamics (
|
||||
id bigserial PRIMARY KEY,
|
||||
house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE,
|
||||
month_date date NOT NULL, -- '2025-11-01' (first of month)
|
||||
price_per_sqm numeric NOT NULL, -- avg price per sqm for that month
|
||||
source text NOT NULL DEFAULT 'cian_realty_valuation',
|
||||
recorded_at timestamptz NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (house_id, month_date, source)
|
||||
);
|
||||
|
||||
-- Primary read pattern: time-ordered series for a house
|
||||
CREATE INDEX IF NOT EXISTS hpd_house_date_idx
|
||||
ON houses_price_dynamics (house_id, month_date DESC);
|
||||
|
||||
COMMIT;
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
-- 025_house_reliability_checks.sql
|
||||
-- Purpose: наш.дом.рф reliability check results scraped from Cian newbuilding pages.
|
||||
-- Stores per-house check status and detail array.
|
||||
-- Dependencies:
|
||||
-- - houses table (009_houses.sql)
|
||||
-- Deploy order: Apply after 024.
|
||||
--
|
||||
-- Sources: Schema_Cian_SERP_Inventory sec 16.4
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS house_reliability_checks (
|
||||
id bigserial PRIMARY KEY,
|
||||
house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE,
|
||||
check_status text, -- 'reliable'/'problematic'/'unchecked'
|
||||
check_name text, -- human-readable check name / title
|
||||
details jsonb, -- [{type, title, iconType, ...}] raw checks array
|
||||
source text NOT NULL DEFAULT 'cian_nashdom',
|
||||
recorded_at timestamptz NOT NULL DEFAULT NOW()
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS hrc_house_idx
|
||||
ON house_reliability_checks (house_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS hrc_status_idx
|
||||
ON house_reliability_checks (check_status)
|
||||
WHERE check_status IS NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
46
tradein-mvp/backend/data/sql/026_external_valuations.sql
Normal file
46
tradein-mvp/backend/data/sql/026_external_valuations.sql
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
-- 026_external_valuations.sql
|
||||
-- Purpose: Unified cache for external valuation APIs — Avito IMV and Cian Valuation Calculator.
|
||||
-- Replaces/supersedes avito_imv_evaluations (018) for multi-source use.
|
||||
-- 018 table kept for backward compat; new code writes here.
|
||||
-- Dependencies:
|
||||
-- - None (standalone cache table)
|
||||
-- Deploy order: Apply after 025.
|
||||
--
|
||||
-- Sources: Schema_Cian_SERP_Inventory sec 26.3 (external_valuations DDL)
|
||||
-- CianScraper_v1_Implementation_Plan Stage 1
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS external_valuations (
|
||||
id bigserial PRIMARY KEY,
|
||||
source text NOT NULL, -- 'avito_imv' / 'cian_valuation'
|
||||
cache_key text NOT NULL, -- sha256(address+area+rooms+floor+repair+deal)
|
||||
-- Input snapshot
|
||||
address text,
|
||||
total_area numeric(8,2),
|
||||
rooms_count int,
|
||||
floor int,
|
||||
total_floors int,
|
||||
repair_type text, -- without/cosmetic/euro/design
|
||||
deal_type text, -- sale/rent
|
||||
-- Output
|
||||
price_rub numeric, -- primary estimate price
|
||||
accuracy numeric, -- accuracy % (Cian-specific; 0-100)
|
||||
raw_payload jsonb, -- full API response
|
||||
chart jsonb, -- [{date, price}, ...] monthly points
|
||||
-- Cache TTL
|
||||
fetched_at timestamptz NOT NULL DEFAULT NOW(),
|
||||
expires_at timestamptz NOT NULL, -- fetched_at + 24h typically
|
||||
UNIQUE (source, cache_key)
|
||||
);
|
||||
|
||||
-- Cache lookup: valid entries only
|
||||
CREATE INDEX IF NOT EXISTS ev_source_expires_idx
|
||||
ON external_valuations (source, expires_at DESC);
|
||||
|
||||
-- Address-based lookup (for analytics / dedup)
|
||||
CREATE INDEX IF NOT EXISTS ev_address_idx
|
||||
ON external_valuations (address)
|
||||
WHERE address IS NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
39
tradein-mvp/backend/data/sql/027_cian_session_cookies.sql
Normal file
39
tradein-mvp/backend/data/sql/027_cian_session_cookies.sql
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
-- 027_cian_session_cookies.sql
|
||||
-- Purpose: Encrypted storage for Cian browser session cookies needed to call
|
||||
-- the Valuation Calculator endpoint (auth required).
|
||||
-- Uses pgcrypto pgp_sym_encrypt for AES encryption at rest.
|
||||
-- Dependencies:
|
||||
-- - pgcrypto extension (installed here via CREATE EXTENSION IF NOT EXISTS)
|
||||
-- Deploy order: Apply after 026.
|
||||
--
|
||||
-- Security notes:
|
||||
-- - cookies_encrypted stores AES-encrypted JSON blob via pgp_sym_encrypt.
|
||||
-- - Encryption key lives in .env.runtime (COOKIE_ENCRYPTION_KEY), never in DB.
|
||||
-- - Access restricted to admin-token-gated API endpoint only.
|
||||
--
|
||||
-- Sources: CianScraper_v1_Implementation_Plan (Cookie auth section)
|
||||
|
||||
BEGIN;
|
||||
|
||||
CREATE EXTENSION IF NOT EXISTS pgcrypto;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS cian_session_cookies (
|
||||
account_user_id bigint PRIMARY KEY, -- Cian user_id extracted from state.user.userId
|
||||
cookies_encrypted bytea NOT NULL, -- pgp_sym_encrypt(json_cookies, key)
|
||||
expires_at_estimate timestamptz NOT NULL, -- estimated expiry (~30 days from upload)
|
||||
uploaded_at timestamptz NOT NULL DEFAULT NOW(),
|
||||
last_used_at timestamptz, -- set on each successful valuation call
|
||||
last_invalid_at timestamptz, -- set when isAuthenticated=false detected
|
||||
notes text -- e.g. 'Account: lekss361@cian.ru'
|
||||
);
|
||||
|
||||
-- Most recent upload first (admin dashboard ordering)
|
||||
CREATE INDEX IF NOT EXISTS cian_cookies_uploaded_idx
|
||||
ON cian_session_cookies (uploaded_at DESC);
|
||||
|
||||
-- Active session lookup (non-expired and not invalidated)
|
||||
CREATE INDEX IF NOT EXISTS cian_cookies_active_idx
|
||||
ON cian_session_cookies (expires_at_estimate)
|
||||
WHERE last_invalid_at IS NULL;
|
||||
|
||||
COMMIT;
|
||||
103
tradein-mvp/backend/data/sql/028_matching_tables.sql
Normal file
103
tradein-mvp/backend/data/sql/028_matching_tables.sql
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
-- 028_matching_tables.sql
|
||||
-- Purpose: Cross-source matching link tables for deduplication and enrichment.
|
||||
-- Implements 3-tier house + apartment matching per Cross_Source_Matching_Strategy.
|
||||
-- house_sources — one canonical house ↔ many external (source, ext_id) pairs
|
||||
-- listing_sources — one canonical listing ↔ many external (source, ext_id) pairs
|
||||
-- house_address_aliases — manual overrides for corpus/address format mismatches
|
||||
-- Dependencies:
|
||||
-- - houses table (009_houses.sql)
|
||||
-- - listings table (002_core_tables.sql)
|
||||
-- Deploy order: Apply after 027.
|
||||
--
|
||||
-- Sources: Cross_Source_Matching_Strategy sec 2
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- address_fingerprint column on houses (needed for Tier-1 matching algorithm)
|
||||
ALTER TABLE houses
|
||||
ADD COLUMN IF NOT EXISTS address_fingerprint text;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS houses_addr_fp_idx
|
||||
ON houses (address_fingerprint)
|
||||
WHERE address_fingerprint IS NOT NULL;
|
||||
|
||||
-- canonical / merged_into fields on listings (needed for dedup workflow)
|
||||
ALTER TABLE listings
|
||||
ADD COLUMN IF NOT EXISTS canonical boolean DEFAULT true,
|
||||
ADD COLUMN IF NOT EXISTS merged_into bigint REFERENCES listings(id) ON DELETE SET NULL;
|
||||
|
||||
CREATE INDEX IF NOT EXISTS listings_merged_into_idx
|
||||
ON listings (merged_into)
|
||||
WHERE merged_into IS NOT NULL;
|
||||
|
||||
-- -----------------------------------------------------------------------
|
||||
-- house_sources: link canonical house to external source records
|
||||
-- -----------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS house_sources (
|
||||
id bigserial PRIMARY KEY,
|
||||
house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE,
|
||||
ext_source text NOT NULL, -- 'avito' / 'cian' / 'yandex' / 'n1'
|
||||
ext_id text NOT NULL, -- source-specific house/newbuilding ID
|
||||
confidence real NOT NULL, -- 0.0–1.0
|
||||
matched_method text NOT NULL, -- 'cadastr_exact'/'address_geo_year'/'geo_only'/'new'/'manual'
|
||||
matched_at timestamptz NOT NULL DEFAULT NOW(),
|
||||
UNIQUE (ext_source, ext_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS house_sources_house_idx
|
||||
ON house_sources (house_id);
|
||||
|
||||
-- Low-confidence matches for review queue
|
||||
CREATE INDEX IF NOT EXISTS house_sources_low_conf_idx
|
||||
ON house_sources (confidence)
|
||||
WHERE confidence < 0.85;
|
||||
|
||||
-- -----------------------------------------------------------------------
|
||||
-- listing_sources: link canonical listing to external source records
|
||||
-- -----------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS listing_sources (
|
||||
id bigserial PRIMARY KEY,
|
||||
listing_id bigint NOT NULL REFERENCES listings(id) ON DELETE CASCADE,
|
||||
ext_source text NOT NULL, -- 'avito' / 'cian'
|
||||
ext_id text NOT NULL, -- Avito source_id / Cian cianId (as text)
|
||||
confidence real NOT NULL, -- 0.0–1.0
|
||||
matched_method text NOT NULL, -- 'cadastr_exact'/'minhash'/'composite'/'new'/'manual'
|
||||
matched_at timestamptz NOT NULL DEFAULT NOW(),
|
||||
source_url text, -- URL объявления на источнике
|
||||
last_scraped_at timestamptz, -- последний раз видели на источнике
|
||||
UNIQUE (ext_source, ext_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS listing_sources_listing_idx
|
||||
ON listing_sources (listing_id);
|
||||
|
||||
-- Low-confidence listing matches for review
|
||||
CREATE INDEX IF NOT EXISTS listing_sources_low_conf_idx
|
||||
ON listing_sources (confidence)
|
||||
WHERE confidence < 0.85;
|
||||
|
||||
-- Recency index for stale-data detection
|
||||
CREATE INDEX IF NOT EXISTS listing_sources_scraped_idx
|
||||
ON listing_sources (ext_source, last_scraped_at DESC);
|
||||
|
||||
-- -----------------------------------------------------------------------
|
||||
-- house_address_aliases: manual mapping for corpus/address format mismatches
|
||||
-- e.g. 'ул. Постовского 17А' ↔ 'ул. Постовского 17 корп.1'
|
||||
-- -----------------------------------------------------------------------
|
||||
CREATE TABLE IF NOT EXISTS house_address_aliases (
|
||||
id bigserial PRIMARY KEY,
|
||||
house_id bigint NOT NULL REFERENCES houses(id) ON DELETE CASCADE,
|
||||
normalized_address text NOT NULL, -- normalized form (output of normalize_address())
|
||||
fingerprint text, -- SHA1 of normalized_address
|
||||
source text, -- which scraper/dataset provided this alias
|
||||
UNIQUE (normalized_address)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS haa_house_idx
|
||||
ON house_address_aliases (house_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS haa_fingerprint_idx
|
||||
ON house_address_aliases (fingerprint)
|
||||
WHERE fingerprint IS NOT NULL;
|
||||
|
||||
COMMIT;
|
||||
Loading…
Add table
Reference in a new issue