From f9b8468bfc146210e01b3477c26d192b47cd4c5a Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 17 May 2026 23:17:59 +0300 Subject: [PATCH] =?UTF-8?q?fix(obj-2):=20trim=20migration=20116=20to=20GIS?= =?UTF-8?q?T=20trgm=20index=20only=20=E2=80=94=20INSERT=20CTE=20timed=20ou?= =?UTF-8?q?t=20in=20deploy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Original CTE (CROSS JOIN LATERAL similarity на 1056 × 320 unmapped × top-3) завис на deploy таймауте (89M cost узнан через EXPLAIN database-expert worker). GIST trgm index — load-bearing для downstream queries, оставлен. Multi-feature INSERT (composite >= 0.75) уже частично применён inline 2026-05-17 (+13 мапingов: 129 → 142). Полный backfill будет в отдельной миграции 117 с pre-materialized temp tables для avoidance of full LATERAL scan. --- data/sql/116_obj2_multi_feature_matcher.sql | 101 ++------------------ 1 file changed, 10 insertions(+), 91 deletions(-) diff --git a/data/sql/116_obj2_multi_feature_matcher.sql b/data/sql/116_obj2_multi_feature_matcher.sql index 434af084..98da53eb 100644 --- a/data/sql/116_obj2_multi_feature_matcher.sql +++ b/data/sql/116_obj2_multi_feature_matcher.sql @@ -1,96 +1,15 @@ --- 116_obj2_multi_feature_matcher.sql --- Issue #307 OBJ-2: backfill objective_complex_mapping via multi-feature scoring --- (name_sim 60% + dev_match 25% + district_match 15%). +-- 116_obj2_multi_feature_matcher.sql (HOTFIX — INSERT step deferred) +-- Issue #307 OBJ-2. -- --- Adds trgm gist index on objective_lots.project_name FIRST so the LATERAL --- similarity() probe runs as bitmap index scan instead of seq-scan-per-row. +-- Original version did INSERT through a 89M-cost CROSS JOIN LATERAL CTE that +-- timed out during deploy. Hotfix keeps только the GIST trgm index (this is +-- the load-bearing change for downstream queries — competitors fuzzy match, +-- objective_complex_mapping reverse matching). -- --- Inserts auto-accepted matches at composite >= 0.75 with match_method='multi_feature_v1'. --- ON CONFLICT (objective_complex_name, objective_group) DO NOTHING — idempotent. --- REFRESH mv_layout_velocity CONCURRENTLY at the end so Site Finder sees new mappings. --- --- Verification (run after COMMIT, outside migration): --- SELECT COUNT(*) FROM objective_complex_mapping; -- expect 200+ --- SELECT * FROM objective_complex_mapping WHERE domrf_obj_id=64701; -- Малевич still mapped --- SELECT match_method, COUNT(*) FROM objective_complex_mapping GROUP BY 1; +-- Actual multi-feature INSERT was already done inline on 2026-05-17 +-- (+13 mappings: 129 → 142). Remaining auto-accept candidates (~50-100 at +-- composite >= 0.75) will be applied via separate migration 117 after +-- the candidate query is pre-materialized to avoid full LATERAL scan. --- The pg_trgm extension is already installed (used by existing migrations 99/103/104). - --- ── Step 1: GIST index for trgm similarity probes ─────────────────────────── --- IF NOT EXISTS keeps migration idempotent. CREATE INDEX IF NOT EXISTS objective_lots_project_name_trgm_idx ON objective_lots USING gist (project_name gist_trgm_ops); - --- ── Step 2: composite-score backfill ──────────────────────────────────────── -BEGIN; - -WITH unmapped_domrf AS ( - SELECT o.obj_id, o.comm_name, o.dev_name, o.district_name - FROM domrf_kn_objects o - WHERE o.is_ekb = true - AND o.comm_name IS NOT NULL - AND NOT EXISTS ( - SELECT 1 FROM objective_complex_mapping m WHERE m.domrf_obj_id = o.obj_id - ) -), -unmapped_obj AS ( - SELECT DISTINCT project_name, developer, district - FROM objective_lots - WHERE project_name IS NOT NULL - AND project_name NOT IN ( - SELECT objective_complex_name FROM objective_complex_mapping - ) -), -pair_scores AS ( - SELECT - d.obj_id, - o.project_name, - similarity(d.comm_name, o.project_name) AS name_sim, - (regexp_replace(lower(coalesce(d.dev_name, '')), '[^а-яa-z0-9]', '', 'g') - = regexp_replace(lower(coalesce(o.developer, '')), '[^а-яa-z0-9]', '', 'g') - )::int AS dev_match, - ( - coalesce(d.district_name, '') ILIKE '%' || coalesce(o.district, '') || '%' - OR coalesce(o.district, '') ILIKE '%' || coalesce(d.district_name, '') || '%' - )::int AS district_match - FROM unmapped_domrf d - CROSS JOIN LATERAL ( - SELECT u.project_name, u.developer, u.district - FROM unmapped_obj u - WHERE similarity(d.comm_name, u.project_name) > 0.3 - ORDER BY similarity(d.comm_name, u.project_name) DESC - LIMIT 3 - ) o -), -ranked AS ( - SELECT - obj_id, project_name, name_sim, dev_match, district_match, - (name_sim * 0.6 + dev_match * 0.25 + district_match * 0.15) AS composite, - ROW_NUMBER() OVER ( - PARTITION BY obj_id - ORDER BY (name_sim * 0.6 + dev_match * 0.25 + district_match * 0.15) DESC, - name_sim DESC - ) AS rn - FROM pair_scores -) -INSERT INTO objective_complex_mapping - (objective_complex_name, domrf_obj_id, objective_group, - match_method, match_score, is_reviewed, note) -SELECT - r.project_name, - r.obj_id, - 'Екатеринбург', - 'multi_feature_v1', - r.composite, - false, - 'OBJ-2 multi-feature 2026-05-17 (name 0.6 + dev 0.25 + district 0.15)' - FROM ranked r - WHERE r.rn = 1 - AND r.composite >= 0.75 -ON CONFLICT (objective_complex_name, objective_group) DO NOTHING; - -COMMIT; - --- ── Step 3: REFRESH MV so Site Finder uses new mappings ───────────────────── --- CONCURRENTLY requires the MV to not be in a tx; run outside BEGIN/COMMIT. -REFRESH MATERIALIZED VIEW CONCURRENTLY mv_layout_velocity; -- 2.45.3