diff --git a/tradein-mvp/backend/data/sql/108_merge_duplicate_houses.sql b/tradein-mvp/backend/data/sql/108_merge_duplicate_houses.sql index beb9992e..e66019af 100644 --- a/tradein-mvp/backend/data/sql/108_merge_duplicate_houses.sql +++ b/tradein-mvp/backend/data/sql/108_merge_duplicate_houses.sql @@ -23,7 +23,7 @@ -- house_imv_evaluations (house_id) — CASCADE — UNIQUE(house_id) → dedup -- house_suggestions (house_id) — CASCADE — UNIQUE(house_id, ext_item_id) → dedup -- external_valuations (house_id) — CASCADE — нет ограничивающего UNIQUE --- address_mismatch_audit (house_id) — CASCADE — нет ограничивающего UNIQUE +-- address_mismatch_audit (house_id) — CASCADE — UNIQUE(house_id,audit_batch) → dedup -- -- Идемпотентность: все UPDATE/DELETE ищут по mapping временной таблицы. При повторном -- запуске mapping будет пустым (дублей нет) → все UPDATE/DELETE затронут 0 строк → no-op. @@ -80,7 +80,7 @@ clusters_with_count AS ( ), -- Только дома, входящие в кластер-дубль dup_houses AS ( - SELECT h.id, c.cluster_key + SELECT h.id, cl.cluster_key FROM houses h JOIN clustered cl ON cl.id = h.id JOIN clusters_with_count cw ON cw.cluster_key = cl.cluster_key @@ -203,47 +203,58 @@ FROM _108_dup_mapping m WHERE haa.house_id = m.dup_id; -- 3c. houses_price_dynamics UNIQUE(house_id, month_date, source) --- Удалить строки дубля, у которых (month_date,source) уже есть у canonical. -DELETE FROM houses_price_dynamics hpd -USING _108_dup_mapping m -WHERE hpd.house_id = m.dup_id - AND EXISTS ( - SELECT 1 FROM houses_price_dynamics hpd2 - WHERE hpd2.house_id = m.canonical_id - AND hpd2.month_date = hpd.month_date - AND hpd2.source = hpd.source - ); --- Репойнтить оставшиеся +-- Collision-safe: дедуп по ЦЕЛЕВОМУ ключу (canonical, month_date, source) — +-- ловит и dup-vs-canonical, и dup-vs-dup в кластерах >2. Выживает строка +-- canonical'а (m.canonical_id IS NULL), иначе минимальный id. +DELETE FROM houses_price_dynamics t +USING ( + SELECT t2.id, + ROW_NUMBER() OVER ( + PARTITION BY COALESCE(m.canonical_id, t2.house_id), t2.month_date, t2.source + ORDER BY (m.canonical_id IS NULL) DESC, t2.id ASC + ) AS rn + FROM houses_price_dynamics t2 + LEFT JOIN _108_dup_mapping m ON m.dup_id = t2.house_id +) d +WHERE t.id = d.id AND d.rn > 1; +-- Репойнтить оставшиеся (целевой ключ теперь свободен) UPDATE houses_price_dynamics hpd SET house_id = m.canonical_id FROM _108_dup_mapping m WHERE hpd.house_id = m.dup_id; --- 3d. house_imv_evaluations UNIQUE(house_id) — у canonical уже может быть запись --- Удалить строки дубля если canonical уже имеет evaluation. -DELETE FROM house_imv_evaluations hie -USING _108_dup_mapping m -WHERE hie.house_id = m.dup_id - AND EXISTS ( - SELECT 1 FROM house_imv_evaluations hie2 - WHERE hie2.house_id = m.canonical_id - ); --- Репойнтить оставшиеся (canonical не имеет evaluation — забираем от дубля) +-- 3d. house_imv_evaluations UNIQUE(house_id) +-- Collision-safe: одна evaluation на canonical. Выживает canonical'а, иначе min id. +DELETE FROM house_imv_evaluations t +USING ( + SELECT t2.id, + ROW_NUMBER() OVER ( + PARTITION BY COALESCE(m.canonical_id, t2.house_id) + ORDER BY (m.canonical_id IS NULL) DESC, t2.id ASC + ) AS rn + FROM house_imv_evaluations t2 + LEFT JOIN _108_dup_mapping m ON m.dup_id = t2.house_id +) d +WHERE t.id = d.id AND d.rn > 1; +-- Репойнтить оставшиеся UPDATE house_imv_evaluations hie SET house_id = m.canonical_id FROM _108_dup_mapping m WHERE hie.house_id = m.dup_id; -- 3e. house_suggestions UNIQUE(house_id, ext_item_id) --- Удалить строки дубля, у которых (ext_item_id) уже есть у canonical. -DELETE FROM house_suggestions hs -USING _108_dup_mapping m -WHERE hs.house_id = m.dup_id - AND EXISTS ( - SELECT 1 FROM house_suggestions hs2 - WHERE hs2.house_id = m.canonical_id - AND hs2.ext_item_id = hs.ext_item_id - ); +-- Collision-safe: дедуп по (canonical, ext_item_id) — ловит dup-vs-dup. +DELETE FROM house_suggestions t +USING ( + SELECT t2.id, + ROW_NUMBER() OVER ( + PARTITION BY COALESCE(m.canonical_id, t2.house_id), t2.ext_item_id + ORDER BY (m.canonical_id IS NULL) DESC, t2.id ASC + ) AS rn + FROM house_suggestions t2 + LEFT JOIN _108_dup_mapping m ON m.dup_id = t2.house_id +) d +WHERE t.id = d.id AND d.rn > 1; -- Репойнтить оставшиеся UPDATE house_suggestions hs SET house_id = m.canonical_id @@ -268,7 +279,20 @@ SET house_id = m.canonical_id FROM _108_dup_mapping m WHERE ev.house_id = m.dup_id; --- 3i. address_mismatch_audit (CASCADE, нет блокирующего UNIQUE) +-- 3i. address_mismatch_audit UNIQUE(house_id, audit_batch) — collision-safe +-- (заголовок ранее ошибочно помечал «нет блокирующего UNIQUE»). +DELETE FROM address_mismatch_audit t +USING ( + SELECT t2.id, + ROW_NUMBER() OVER ( + PARTITION BY COALESCE(m.canonical_id, t2.house_id), t2.audit_batch + ORDER BY (m.canonical_id IS NULL) DESC, t2.id ASC + ) AS rn + FROM address_mismatch_audit t2 + LEFT JOIN _108_dup_mapping m ON m.dup_id = t2.house_id +) d +WHERE t.id = d.id AND d.rn > 1; +-- Репойнтить оставшиеся UPDATE address_mismatch_audit ama SET house_id = m.canonical_id FROM _108_dup_mapping m