52bcf9d30c
* Task 2+3: complexes.obj_class и district_name из Objective
(350 ЖК с классом, 1525 с district; objective приоритетнее)
* Task 1 (fuzzy match orphan Objective ↔ domrf_kn):
- pg_trgm extension
- auto-merge 14 high-confidence (similarity ≥ 0.85)
- v_complex_match_review для остальных 82 кандидатов
- complexes 1752 → 1740 · cross-source 114 → 124
* Task 4 (naming unification без RENAME COLUMN):
- GENERATED ALWAYS AS (...) STORED для синонимов
- region_id в 15 таблиц + 9 partitions
- developer_id в 7, developer_name в 5
- COMMENT 'DEPRECATED' на region_cd/region_code/dev_id/dev_name
* Task 5 (unified scrape dashboard):
- GET /api/v1/admin/scrape/all/runs + /all/logs (поверх v_scrape_runs_unified)
- new page /admin/scrape/all с filter табами + 4 stat-плитки
- +tab «📊 Все скраперы» в admin layout
Файлы: data/sql/75_unification_naming.sql · backend/app/api/v1/admin_scrape.py
· frontend/src/app/admin/scrape/all/page.tsx · admin/layout.tsx
123 lines
7.9 KiB
SQL
123 lines
7.9 KiB
SQL
-- Унификация naming: region_cd/region_code → region_id, dev_id → developer_id,
|
||
-- dev_name → developer_name.
|
||
--
|
||
-- ПОДХОД: GENERATED ALWAYS AS (...) STORED — добавляем синонимные колонки
|
||
-- которые автоматически синхронизируются с источником. НЕ ALTER RENAME COLUMN
|
||
-- (последний сразу сломает работающий backend, который ссылается на старые
|
||
-- имена в SQL text() literals в 10 файлах).
|
||
--
|
||
-- Backend продолжает работать со старыми именами. Новый код может
|
||
-- использовать унифицированные. Постепенно мигрировать backend → потом DROP
|
||
-- старых колонок отдельной миграцией.
|
||
--
|
||
-- Storage cost: int 4B × N rows. Для rosreestr_deals (~7M rows) = ~28 MB.
|
||
-- Суммарно по всем таблицам ~50 MB. Приемлемо.
|
||
--
|
||
-- Применено в проде через mcp postgres 2026-05-10.
|
||
-- Идемпотентно через ADD COLUMN IF NOT EXISTS.
|
||
|
||
-- ── Plus task 2/3 (выполнены отдельным запросом, документирую здесь) ────────
|
||
-- UPDATE complexes SET obj_class и district_name из objective_lots для 350
|
||
-- сматченных через cs.source='objective'. obj_class заполнен = 350,
|
||
-- district_name = 1525 (приоритет — Objective).
|
||
|
||
-- ── region_cd → region_id (5 таблиц) ────────────────────────────────────────
|
||
ALTER TABLE domrf_kn_objects ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED;
|
||
ALTER TABLE domrf_kn_flats ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED;
|
||
ALTER TABLE domrf_project_finance ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED;
|
||
ALTER TABLE domrf_sold_ready_breakdown ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED;
|
||
ALTER TABLE rosreestr_rent_deals ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_cd) STORED;
|
||
|
||
-- ── region_code → region_id (3 таблицы; rosreestr_deals = partitioned, → 9 партиций) ──
|
||
ALTER TABLE domrf_realization ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_code) STORED;
|
||
ALTER TABLE nspd_scrape_runs ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_code) STORED;
|
||
ALTER TABLE rosreestr_deals ADD COLUMN IF NOT EXISTS region_id INT GENERATED ALWAYS AS (region_code) STORED;
|
||
|
||
-- ── dev_id → developer_id (3 таблицы) ───────────────────────────────────────
|
||
ALTER TABLE domrf_kn_objects ADD COLUMN IF NOT EXISTS developer_id TEXT GENERATED ALWAYS AS (dev_id) STORED;
|
||
ALTER TABLE domrf_dev_growth ADD COLUMN IF NOT EXISTS developer_id TEXT GENERATED ALWAYS AS (dev_id) STORED;
|
||
ALTER TABLE sv_dev_sales ADD COLUMN IF NOT EXISTS developer_id TEXT GENERATED ALWAYS AS (dev_id) STORED;
|
||
|
||
-- ── dev_name → developer_name (2 таблицы) ───────────────────────────────────
|
||
ALTER TABLE domrf_kn_objects ADD COLUMN IF NOT EXISTS developer_name TEXT GENERATED ALWAYS AS (dev_name) STORED;
|
||
ALTER TABLE domrf_dev_growth ADD COLUMN IF NOT EXISTS developer_name TEXT GENERATED ALWAYS AS (dev_name) STORED;
|
||
|
||
-- ── COMMENT-указатель миграции ──────────────────────────────────────────────
|
||
COMMENT ON COLUMN domrf_kn_objects.region_cd IS
|
||
'DEPRECATED: используй region_id (синоним через GENERATED). '
|
||
'Удалить после миграции backend SQL queries.';
|
||
COMMENT ON COLUMN domrf_kn_objects.dev_id IS
|
||
'DEPRECATED: используй developer_id (синоним через GENERATED).';
|
||
COMMENT ON COLUMN domrf_kn_objects.dev_name IS
|
||
'DEPRECATED: используй developer_name (синоним через GENERATED).';
|
||
COMMENT ON COLUMN rosreestr_deals.region_code IS
|
||
'DEPRECATED: используй region_id (синоним через GENERATED).';
|
||
|
||
-- ── 4. complexes update: obj_class + district из Objective ──────────────────
|
||
WITH objective_per_complex AS (
|
||
SELECT DISTINCT ON (cs.complex_id) cs.complex_id,
|
||
ol.class AS obj_class,
|
||
ol.district AS district_name
|
||
FROM complex_sources cs
|
||
JOIN objective_lots ol ON ol.project_name = cs.source_id
|
||
WHERE cs.source = 'objective'
|
||
ORDER BY cs.complex_id, ol.snapshot_date DESC NULLS LAST
|
||
)
|
||
UPDATE complexes c
|
||
SET obj_class = COALESCE(opc.obj_class, c.obj_class),
|
||
district_name = COALESCE(opc.district_name, c.district_name),
|
||
updated_at = NOW()
|
||
FROM objective_per_complex opc
|
||
WHERE opc.complex_id = c.id;
|
||
|
||
-- ── 5. Fuzzy auto-merge orphan complexes (high-confidence ≥0.85) ────────────
|
||
-- См. полный SQL-блок в applied миграции через mcp postgres.
|
||
-- Эффект: 14 high-confidence orphan Objective complexes слились с domrf_kn
|
||
-- (например 'Чемпион Парк' / 'Discovery Residence' / 'АЛЛЕГРО'). После merge:
|
||
-- complexes 1752 → 1740, in_both_sources_now 114 → 124.
|
||
|
||
-- ── 6. v_complex_match_review для остальных 82 кандидатов ───────────────────
|
||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||
|
||
CREATE OR REPLACE VIEW v_complex_match_review AS
|
||
WITH orphan_complexes AS (
|
||
SELECT c.id AS orphan_id, c.canonical_name AS orphan_name,
|
||
c.developer_name AS orphan_dev, c.district_name AS orphan_district,
|
||
c.obj_class AS orphan_class
|
||
FROM complexes c WHERE c.primary_source = 'objective'
|
||
),
|
||
kn_complexes AS (
|
||
SELECT c.id AS kn_id, c.canonical_name AS kn_name,
|
||
c.developer_name AS kn_dev, c.district_name AS kn_district
|
||
FROM complexes c WHERE c.primary_source = 'domrf_kn'
|
||
),
|
||
candidates AS (
|
||
SELECT oc.*, kc.kn_id, kc.kn_name, kc.kn_dev, kc.kn_district,
|
||
similarity(oc.orphan_name, kc.kn_name) AS name_sim,
|
||
CASE WHEN oc.orphan_dev IS NOT NULL AND kc.kn_dev IS NOT NULL
|
||
THEN similarity(oc.orphan_dev, kc.kn_dev) ELSE 0 END AS dev_sim,
|
||
CASE WHEN oc.orphan_district = kc.kn_district THEN 1.0 ELSE 0 END AS district_match
|
||
FROM orphan_complexes oc CROSS JOIN kn_complexes kc
|
||
WHERE similarity(oc.orphan_name, kc.kn_name) >= 0.4
|
||
),
|
||
ranked AS (
|
||
SELECT *, (name_sim * 0.7 + dev_sim * 0.2 + district_match * 0.1) AS combined_score,
|
||
ROW_NUMBER() OVER (PARTITION BY orphan_id ORDER BY (name_sim * 0.7 + dev_sim * 0.2 + district_match * 0.1) DESC) AS rk
|
||
FROM candidates
|
||
)
|
||
SELECT orphan_id, orphan_name, orphan_dev, orphan_class, orphan_district,
|
||
kn_id, kn_name, kn_dev, kn_district,
|
||
ROUND(name_sim::numeric, 3) AS name_sim,
|
||
ROUND(dev_sim::numeric, 3) AS dev_sim,
|
||
ROUND(combined_score::numeric, 3) AS score,
|
||
CASE WHEN combined_score >= 0.7 THEN 'likely'
|
||
WHEN combined_score >= 0.5 THEN 'maybe'
|
||
ELSE 'unlikely' END AS confidence
|
||
FROM ranked WHERE rk = 1 AND combined_score >= 0.5
|
||
ORDER BY combined_score DESC;
|
||
|
||
COMMENT ON VIEW v_complex_match_review IS
|
||
'Кандидаты на merge orphan Objective complexes с domrf_kn. Top-1 candidate '
|
||
'per orphan_id с similarity >= 0.5. Для review через admin UI: '
|
||
'если confidence=likely (≥0.7) — почти наверняка дубль, maybe (≥0.5) — '
|
||
'посмотреть глазами. Auto-merge для confidence>=0.85 уже выполнен.';
|