542ff1c9ca
Оба места утверждали, что у Домклика один выделенный резидентный прокси и пула нет. Это перестало быть правдой ещё в #2800 (миграция 253 сняла резервацию узла), но текст остался — и именно на него опирался тикет #3189, поставленный под «калибровку» ограничения, которого не существует. Свип к тому же ходит через пул давно (serp.py:359), а бэкфилл подключён к нему в PR #3222. Заодно докстринг называл не тот ограничитель: свип кладёт не счётчик провалов lease, а break по первому DomClickBlockedError (#2854) — до ротации дело не доходит ни при каком счётчике, отсюда buckets_completed=0. Только комментарии, поведение не меняется. Миграция 175 уже применена, а _schema_migrations трекает по имени файла без checksum — правка текста её не перезапустит.
91 lines
3.3 KiB
PL/PgSQL
91 lines
3.3 KiB
PL/PgSQL
-- 117_obj2_mapping_backfill_fast.sql
|
||
-- Issue #307 OBJ-2 (continued from 116 hotfix).
|
||
--
|
||
-- Migration 116 был trimmed до GIST index only из-за heavy CTE timeout.
|
||
-- Эта migration делает фактический INSERT через **pre-materialized temp tables**
|
||
-- (avoid full CROSS JOIN LATERAL scan). Composite score = name 0.6 + dev 0.25 + district 0.15.
|
||
-- Threshold composite >= 0.75. ON CONFLICT DO NOTHING (idempotent).
|
||
--
|
||
-- Estimated runtime после оптимизации: <30s (vs old 89M-cost timeout).
|
||
-- Coverage цель: +50-80 mappings → объединённый total ~200.
|
||
|
||
BEGIN;
|
||
|
||
-- Pre-materialize unmapped DOM.РФ ЕКБ objects (~1056 rows)
|
||
CREATE TEMP TABLE _u_domrf ON COMMIT DROP AS
|
||
SELECT o.obj_id, o.comm_name, o.dev_name, o.district_name,
|
||
regexp_replace(lower(coalesce(o.dev_name, '')), '[^а-яa-z0-9]', '', 'g') AS dev_norm
|
||
FROM domrf_kn_objects o
|
||
WHERE o.is_ekb = true
|
||
AND o.comm_name IS NOT NULL
|
||
AND NOT EXISTS (
|
||
SELECT 1 FROM objective_complex_mapping m WHERE m.domrf_obj_id = o.obj_id
|
||
);
|
||
|
||
-- Pre-materialize distinct unmapped Objective projects (~178 rows)
|
||
CREATE TEMP TABLE _u_obj ON COMMIT DROP AS
|
||
SELECT DISTINCT
|
||
l.project_name,
|
||
l.developer,
|
||
l.district,
|
||
regexp_replace(lower(coalesce(l.developer, '')), '[^а-яa-z0-9]', '', 'g') AS dev_norm
|
||
FROM objective_lots l
|
||
WHERE l.project_name IS NOT NULL
|
||
AND l.project_name NOT IN (
|
||
SELECT objective_complex_name FROM objective_complex_mapping
|
||
);
|
||
|
||
-- Top-3 candidates per DOM.РФ obj_id using GIST trgm index (from migration 116)
|
||
CREATE TEMP TABLE _pairs ON COMMIT DROP AS
|
||
SELECT
|
||
d.obj_id,
|
||
o.project_name,
|
||
o.developer,
|
||
o.district,
|
||
similarity(d.comm_name, o.project_name) AS name_sim,
|
||
(d.dev_norm = o.dev_norm AND d.dev_norm <> '')::int AS dev_match,
|
||
(
|
||
coalesce(d.district_name, '') ILIKE '%' || coalesce(o.district, '') || '%'
|
||
OR coalesce(o.district, '') ILIKE '%' || coalesce(d.district_name, '') || '%'
|
||
)::int AS district_match
|
||
FROM _u_domrf d
|
||
CROSS JOIN LATERAL (
|
||
SELECT u.project_name, u.developer, u.district, u.dev_norm
|
||
FROM _u_obj u
|
||
WHERE similarity(d.comm_name, u.project_name) > 0.3
|
||
ORDER BY similarity(d.comm_name, u.project_name) DESC
|
||
LIMIT 3
|
||
) o;
|
||
|
||
-- Rank per obj_id by composite score, pick top
|
||
WITH ranked AS (
|
||
SELECT
|
||
obj_id, project_name,
|
||
(name_sim * 0.6 + dev_match * 0.25 + district_match * 0.15) AS composite,
|
||
ROW_NUMBER() OVER (
|
||
PARTITION BY obj_id
|
||
ORDER BY (name_sim * 0.6 + dev_match * 0.25 + district_match * 0.15) DESC,
|
||
name_sim DESC
|
||
) AS rn
|
||
FROM _pairs
|
||
)
|
||
INSERT INTO objective_complex_mapping
|
||
(objective_complex_name, domrf_obj_id, objective_group,
|
||
match_method, match_score, is_reviewed, note)
|
||
SELECT
|
||
r.project_name,
|
||
r.obj_id,
|
||
'Екатеринбург',
|
||
'multi_feature_v1',
|
||
r.composite,
|
||
false,
|
||
'OBJ-2 multi-feature 117_fast 2026-05-17 (name 0.6 + dev 0.25 + district 0.15)'
|
||
FROM ranked r
|
||
WHERE r.rn = 1
|
||
AND r.composite >= 0.75
|
||
ON CONFLICT (objective_complex_name, objective_group) DO NOTHING;
|
||
|
||
COMMIT;
|
||
|
||
-- REFRESH MV outside tx (CONCURRENTLY requires non-tx)
|
||
REFRESH MATERIALIZED VIEW CONCURRENTLY mv_layout_velocity;
|