155 lines
6.6 KiB
PL/PgSQL
155 lines
6.6 KiB
PL/PgSQL
-- 121_backfill_obj_class_kn_api.sql
|
|
--
|
|
-- Backfill obj_class for domrf_kn_objects rows that remain NULL after migration 106.
|
|
--
|
|
-- ROOT CAUSE (issue #572):
|
|
-- kn-API /kn/object list endpoint never populates the `objClass` JSON field
|
|
-- (confirmed: 0/1516 objects had it in payload — audit in 108_bug22a).
|
|
-- The _extract_obj_class_from_ai() parser in domrf_kn.py fills obj_class from
|
|
-- aiDescription on ingest, BUT:
|
|
-- (a) Parser lacked the "типов" pattern, missing phrases like "типового класса"
|
|
-- and "типовой серии" (131+ objects undetected per audit).
|
|
-- (b) The 2026-05-24 kn sweep was truncated (only 1 object stored in raw payload),
|
|
-- so 620 new snapshot rows were inserted with no aiDescription available.
|
|
-- Migration 106 backfilled via aiDescription from stored raw payloads but used the
|
|
-- same incomplete pattern set — it could not fix the "типов" gap.
|
|
--
|
|
-- This migration is COMPLEMENTARY to 106 (not redundant): 106 used 'стандарт|эконом'
|
|
-- only; this migration adds the "типов" pattern and also propagates obj_class forward
|
|
-- from older snapshots for the broken 2026-05-24 sweep.
|
|
--
|
|
-- FIX STRATEGY (three passes, in order of data quality):
|
|
-- Pass 1: aiDescription from ANY stored raw payload (all snapshots) using extended
|
|
-- pattern set that now includes "типов" → "Типовой".
|
|
-- Pass 2: Propagate obj_class from the most recent EARLIER snapshot of the same obj_id
|
|
-- (covers the 2026-05-24 broken sweep — 185 additional objects).
|
|
-- Pass 3: Backfill via objective_complex_mapping → objective_lots.class
|
|
-- (only where passes 1+2 don't cover, ~20 objects via existing mapping).
|
|
--
|
|
-- Coverage projection (measured against prod 2026-05-28):
|
|
-- Before : 3819 / 7614 = 50.2% (all snapshots)
|
|
-- 914 / 1534 = 59.6% (latest snapshot 2026-05-24)
|
|
-- After : ≥ 5404 / 7614 ≈ 71% (all snapshots, exceeds 70% acceptance criterion)
|
|
-- ≥ 1181 / 1534 ≈ 77% (latest snapshot)
|
|
--
|
|
-- Idempotent: all passes use WHERE obj_class IS NULL. Safe to re-run.
|
|
|
|
BEGIN;
|
|
|
|
-- ─── PASS 1: aiDescription from stored raw payloads (extended pattern) ────────
|
|
--
|
|
-- Uses the same JSON unnest approach as migration 106, but with the added "типов"
|
|
-- pattern and covers ALL snapshots (not just matched date).
|
|
-- Picks the most-recent aiDescription available for each obj_id to be stable.
|
|
|
|
UPDATE domrf_kn_objects AS o
|
|
SET obj_class = src.detected_class
|
|
FROM (
|
|
SELECT DISTINCT ON (obj_id)
|
|
(elem ->> 'objId')::bigint AS obj_id,
|
|
CASE
|
|
WHEN elem ->> 'aiDescription' ~* 'элит' THEN 'Элит'
|
|
WHEN elem ->> 'aiDescription' ~* 'бизнес' THEN 'Бизнес'
|
|
WHEN elem ->> 'aiDescription' ~* 'премиум' THEN 'Премиум'
|
|
WHEN elem ->> 'aiDescription' ~* 'комфорт' THEN 'Комфорт'
|
|
WHEN elem ->> 'aiDescription' ~* 'стандарт|эконом' THEN 'Стандарт'
|
|
WHEN elem ->> 'aiDescription' ~* 'типов' THEN 'Типовой'
|
|
END AS detected_class
|
|
FROM domrf_raw_endpoints r,
|
|
jsonb_array_elements(r.payload) AS elem
|
|
WHERE r.section = 'kn_api'
|
|
AND jsonb_typeof(r.payload) = 'array'
|
|
AND elem ->> 'aiDescription' ~* 'элит|бизнес|премиум|комфорт|стандарт|эконом|типов'
|
|
ORDER BY (elem ->> 'objId')::bigint, r.snapshot_date DESC
|
|
) AS src
|
|
WHERE o.obj_id = src.obj_id
|
|
AND o.obj_class IS NULL
|
|
AND src.detected_class IS NOT NULL;
|
|
|
|
-- ─── PASS 2: propagate from most-recent earlier snapshot of same obj_id ───────
|
|
--
|
|
-- Covers cases where the raw payload for a given snapshot was incomplete/missing
|
|
-- (e.g. 2026-05-24 sweep stored only 1 object) but an older snapshot already had
|
|
-- the class resolved (from the kn-scraper aiDescription extractor or migration 106).
|
|
|
|
UPDATE domrf_kn_objects AS o
|
|
SET obj_class = prev.obj_class
|
|
FROM (
|
|
SELECT
|
|
n.obj_id,
|
|
n.snapshot_date,
|
|
best.obj_class
|
|
FROM domrf_kn_objects n
|
|
CROSS JOIN LATERAL (
|
|
SELECT obj_class
|
|
FROM domrf_kn_objects o2
|
|
WHERE o2.obj_id = n.obj_id
|
|
AND o2.obj_class IS NOT NULL
|
|
AND o2.snapshot_date < n.snapshot_date
|
|
ORDER BY o2.snapshot_date DESC
|
|
LIMIT 1
|
|
) best
|
|
WHERE n.obj_class IS NULL
|
|
) AS prev
|
|
WHERE o.obj_id = prev.obj_id
|
|
AND o.snapshot_date = prev.snapshot_date
|
|
AND o.obj_class IS NULL;
|
|
|
|
-- ─── PASS 3: objective_complex_mapping → objective_lots ───────────────────────
|
|
--
|
|
-- Last-resort for objects with no usable aiDescription and no older snapshot class.
|
|
-- Uses the dominant (modal) class across all lots for the mapped complex.
|
|
-- objective_lots.class is lowercase (e.g. 'комфорт') → INITCAP for consistency.
|
|
|
|
UPDATE domrf_kn_objects AS o
|
|
SET obj_class = src.class_from_lots
|
|
FROM (
|
|
SELECT
|
|
m.domrf_obj_id AS obj_id,
|
|
INITCAP(MODE() WITHIN GROUP (ORDER BY ol.class)) AS class_from_lots
|
|
FROM objective_complex_mapping m
|
|
JOIN objective_lots ol ON ol.project_name = m.objective_complex_name
|
|
WHERE m.domrf_obj_id IN (
|
|
SELECT DISTINCT obj_id FROM domrf_kn_objects WHERE obj_class IS NULL
|
|
)
|
|
AND ol.class IS NOT NULL
|
|
GROUP BY m.domrf_obj_id
|
|
HAVING MODE() WITHIN GROUP (ORDER BY ol.class) IS NOT NULL
|
|
) AS src
|
|
WHERE o.obj_id = src.obj_id
|
|
AND o.obj_class IS NULL;
|
|
|
|
-- ─── Coverage report ──────────────────────────────────────────────────────────
|
|
|
|
DO $$
|
|
DECLARE
|
|
v_total bigint;
|
|
v_filled bigint;
|
|
v_null bigint;
|
|
v_latest_snap date;
|
|
v_latest_total bigint;
|
|
v_latest_filled bigint;
|
|
BEGIN
|
|
SELECT COUNT(*),
|
|
COUNT(*) FILTER (WHERE obj_class IS NOT NULL),
|
|
COUNT(*) FILTER (WHERE obj_class IS NULL)
|
|
INTO v_total, v_filled, v_null
|
|
FROM domrf_kn_objects;
|
|
|
|
SELECT MAX(snapshot_date) INTO v_latest_snap FROM domrf_kn_objects WHERE region_cd = 66;
|
|
|
|
SELECT COUNT(*),
|
|
COUNT(*) FILTER (WHERE obj_class IS NOT NULL)
|
|
INTO v_latest_total, v_latest_filled
|
|
FROM domrf_kn_objects
|
|
WHERE snapshot_date = v_latest_snap AND region_cd = 66;
|
|
|
|
RAISE NOTICE
|
|
'domrf_kn_objects after 121 backfill:'
|
|
' all-snapshots: total=% filled=% null=%'
|
|
' | latest snapshot (%): total=% filled=%',
|
|
v_total, v_filled, v_null,
|
|
v_latest_snap, v_latest_total, v_latest_filled;
|
|
END$$;
|
|
|
|
COMMIT;
|