Light1YT
86e9ea2937
Многоагентный аудит + имплементация: один воркер на файл, точечные правки. Верификация: py_compile (47/47 .py) + tsc --noEmit (0 ошибок). Unit-тесты не прогонялись (окружение не поднято: rollup native dep / нет pytest-venv). Полностью исправлено (169): #1336, #1337, #1339, #1340, #1341, #1342, #1343, #1345, #1346, #1348, #1349, #1350, #1351, #1354, #1356, #1358, #1359, #1360, #1362, #1364, #1365, #1366, #1367, #1368, #1369, #1370, #1371, #1372, #1373, #1374, #1375, #1376, #1377, #1378, #1379, #1380, #1381, #1382, #1384, #1385, #1386, #1387, #1388, #1389, #1390, #1391, #1392, #1394, #1395, #1396, #1397, #1399, #1400, #1401, #1402, #1403, #1404, #1408, #1409, #1410, #1411, #1412, #1413, #1414, #1415, #1416, #1417, #1418, #1420, #1423, #1425, #1426, #1427, #1428, #1429, #1430, #1431, #1432, #1433, #1434, #1435, #1437, #1438, #1439, #1440, #1441, #1442, #1443, #1444, #1445, #1446, #1447, #1448, #1449, #1450, #1451, #1452, #1453, #1454, #1455, #1456, #1457, #1458, #1459, #1460, #1461, #1462, #1463, #1464, #1465, #1466, #1467, #1468, #1469, #1471, #1472, #1473, #1474, #1476, #1478, #1479, #1481, #1482, #1483, #1484, #1485, #1487, #1488, #1489, #1490, #1491, #1492, #1493, #1494, #1495, #1496, #1497, #1499, #1500, #1501, #1502, #1504, #1505, #1506, #1507, #1510, #1514, #1515, #1516, #1517, #1518, #1519, #1521, #1522, #1523, #1524, #1525, #1526, #1527, #1528, #1529, #1531, #1532, #1533, #1534, #1535, #1536, #1537, #1538 Частично (9, in-file часть, остаток cross-file): #1361, #1419, #1422, #1424, #1470, #1475, #1477, #1480, #1498 Требуют cross-file (3, не тронуты): #1338, #1363, #1421 Пропущено (1): #1539 Не входило в партию: 22 needs-Leha issue (нужны решения владельца). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
51 lines
2.2 KiB
PL/PgSQL
51 lines
2.2 KiB
PL/PgSQL
-- 100_22j_dedup_infrastructure.sql
|
|
-- Context: domrf_kn_infrastructure has ~496k rows but only ~229k unique by
|
|
-- (obj_id, poi_category, poi_name, poi_address) — ~267k duplicates.
|
|
-- Root cause: old scraper re-inserts same POIs on every run with new
|
|
-- snapshot_date, and the old UNIQUE key includes snapshot_date so
|
|
-- dedup was per-day only.
|
|
-- Fix:
|
|
-- 1. One-time cleanup: keep lowest ctid per dedup key, delete the rest.
|
|
-- 2. Drop old UNIQUE constraint (includes snapshot_date — wrong granularity).
|
|
-- 3. Add new UNIQUE (obj_id, poi_category, poi_name, poi_address)
|
|
-- with NULLS NOT DISTINCT (PG16) so NULL values are treated as equal.
|
|
-- Dependencies: none (no views reference this table by constraint name)
|
|
-- Deploy order: apply this SQL first, then deploy backend with ON CONFLICT fix.
|
|
-- Idempotent: yes — constraint uses IF NOT EXISTS equivalent via DO $$ block;
|
|
-- DELETE is safe to re-run (no rows left to delete after first run).
|
|
-- Issue: #297 / task 22j
|
|
-- Applied: 2026-05-17
|
|
|
|
BEGIN;
|
|
|
|
-- Step 1: one-time dedup — keep min(ctid) per logical key
|
|
DELETE FROM domrf_kn_infrastructure a
|
|
USING domrf_kn_infrastructure b
|
|
WHERE a.ctid > b.ctid
|
|
AND a.obj_id = b.obj_id
|
|
AND COALESCE(a.poi_category, '') = COALESCE(b.poi_category, '')
|
|
AND COALESCE(a.poi_name, '') = COALESCE(b.poi_name, '')
|
|
AND COALESCE(a.poi_address, '') = COALESCE(b.poi_address, '');
|
|
|
|
-- Step 2: drop old unique constraint (wrong key — includes snapshot_date)
|
|
ALTER TABLE domrf_kn_infrastructure
|
|
DROP CONSTRAINT IF EXISTS domrf_kn_infrastructure_obj_id_poi_name_poi_lat_poi_lon_sna_key;
|
|
|
|
-- Step 3: add new unique constraint with NULLS NOT DISTINCT (PG15+)
|
|
-- so NULL poi_category / poi_address are treated as equal (not distinct)
|
|
-- PostgreSQL has no ADD CONSTRAINT ... IF NOT EXISTS form, so guard via DO $$
|
|
-- block (idempotency for DR-restore / re-apply under ON_ERROR_STOP).
|
|
DO $$
|
|
BEGIN
|
|
IF NOT EXISTS (
|
|
SELECT 1 FROM pg_constraint
|
|
WHERE conname = 'uq_infra_dedupe'
|
|
AND conrelid = 'domrf_kn_infrastructure'::regclass
|
|
) THEN
|
|
ALTER TABLE domrf_kn_infrastructure
|
|
ADD CONSTRAINT uq_infra_dedupe
|
|
UNIQUE NULLS NOT DISTINCT (obj_id, poi_category, poi_name, poi_address);
|
|
END IF;
|
|
END $$;
|
|
|
|
COMMIT;
|