fix(22j): DEDUP domrf_kn_infrastructure + ON CONFLICT (issue #297) (#304)

backend/app/services/scrapers/domrf_kn.py

@@ -593,11 +593,9 @@ UPSERT_INFRA_SQL = text(
         :obj_id, :poi_name, :poi_subtitle, :poi_category, :poi_address,
         :poi_lat, :poi_lon, :distance_m, :snapshot_date
     )
-    ON CONFLICT (obj_id, poi_name, poi_lat, poi_lon, snapshot_date) DO UPDATE SET
-        poi_subtitle = EXCLUDED.poi_subtitle,
-        poi_category = EXCLUDED.poi_category,
-        poi_address = EXCLUDED.poi_address,
-        distance_m = EXCLUDED.distance_m
+    -- Issue #297 22j: новый UNIQUE (obj_id, poi_category, poi_name, poi_address) — без
+    -- snapshot_date, чтобы каждый scrape run не накапливал ×N дубликаты POI.
+    ON CONFLICT (obj_id, poi_category, poi_name, poi_address) DO NOTHING
     """
 )
 

data/sql/110_22j_dedup_infrastructure.sql

@@ -0,0 +1,40 @@
+-- 100_22j_dedup_infrastructure.sql
+-- Context: domrf_kn_infrastructure has ~496k rows but only ~229k unique by
+--          (obj_id, poi_category, poi_name, poi_address) — ~267k duplicates.
+--          Root cause: old scraper re-inserts same POIs on every run with new
+--          snapshot_date, and the old UNIQUE key includes snapshot_date so
+--          dedup was per-day only.
+-- Fix:
+--   1. One-time cleanup: keep lowest ctid per dedup key, delete the rest.
+--   2. Drop old UNIQUE constraint (includes snapshot_date — wrong granularity).
+--   3. Add new UNIQUE (obj_id, poi_category, poi_name, poi_address)
+--      with NULLS NOT DISTINCT (PG16) so NULL values are treated as equal.
+-- Dependencies: none (no views reference this table by constraint name)
+-- Deploy order: apply this SQL first, then deploy backend with ON CONFLICT fix.
+-- Idempotent: yes — constraint uses IF NOT EXISTS equivalent via DO $$ block;
+--             DELETE is safe to re-run (no rows left to delete after first run).
+-- Issue: #297 / task 22j
+-- Applied: 2026-05-17
+
+BEGIN;
+
+-- Step 1: one-time dedup — keep min(ctid) per logical key
+DELETE FROM domrf_kn_infrastructure a
+USING domrf_kn_infrastructure b
+WHERE a.ctid > b.ctid
+  AND a.obj_id = b.obj_id
+  AND COALESCE(a.poi_category, '') = COALESCE(b.poi_category, '')
+  AND COALESCE(a.poi_name, '') = COALESCE(b.poi_name, '')
+  AND COALESCE(a.poi_address, '') = COALESCE(b.poi_address, '');
+
+-- Step 2: drop old unique constraint (wrong key — includes snapshot_date)
+ALTER TABLE domrf_kn_infrastructure
+  DROP CONSTRAINT IF EXISTS domrf_kn_infrastructure_obj_id_poi_name_poi_lat_poi_lon_sna_key;
+
+-- Step 3: add new unique constraint with NULLS NOT DISTINCT (PG15+)
+--         so NULL poi_category / poi_address are treated as equal (not distinct)
+ALTER TABLE domrf_kn_infrastructure
+  ADD CONSTRAINT uq_infra_dedupe
+  UNIQUE NULLS NOT DISTINCT (obj_id, poi_category, poi_name, poi_address);
+
+COMMIT;