3daf2a1fb1
Add _extract_obj_class_from_ai() with regex patterns for Комфорт/Бизнес/Премиум/Элит/Стандарт. _norm_object() uses it as fallback; objClass from API still takes priority if ever returned. Add backfill SQL 105 to retroactively populate existing rows from raw JSON payloads. Coverage: ~1806/4548 rows will be filled; remainder NULL is expected (Нежилое/no AI desc).
68 lines
2.5 KiB
PL/PgSQL
68 lines
2.5 KiB
PL/PgSQL
-- 105_backfill_obj_class_from_ai_description.sql
|
|
--
|
|
-- Backfill obj_class for existing domrf_kn_objects rows using aiDescription
|
|
-- from the raw domrf_raw_endpoints payloads.
|
|
--
|
|
-- Root cause: DOM.РФ /kn/object list endpoint never returns `objClass` field
|
|
-- (absent from API payload). Class text is embedded in `aiDescription`.
|
|
-- Fix in scraper (domrf_kn.py _norm_object) extracts class on next sweep.
|
|
-- This script retroactively populates existing rows from stored raw payloads.
|
|
--
|
|
-- Coverage after backfill (region 66, snapshot 2026-05-05):
|
|
-- Комфорт: ~782, Бизнес: ~88, Премиум: ~13, Элит: ~12
|
|
-- class_not_found (нежилое / нет AI-описания): ~621 — remains NULL (expected)
|
|
--
|
|
-- Idempotent: only updates WHERE obj_class IS NULL. Safe to re-run.
|
|
|
|
BEGIN;
|
|
|
|
UPDATE domrf_kn_objects AS o
|
|
SET obj_class = CASE
|
|
WHEN raw_obj ->> 'aiDescription' ~* 'элит' THEN 'Элит'
|
|
WHEN raw_obj ->> 'aiDescription' ~* 'бизнес' THEN 'Бизнес'
|
|
WHEN raw_obj ->> 'aiDescription' ~* 'премиум' THEN 'Премиум'
|
|
WHEN raw_obj ->> 'aiDescription' ~* 'комфорт' THEN 'Комфорт'
|
|
WHEN raw_obj ->> 'aiDescription' ~* 'стандарт|эконом' THEN 'Стандарт'
|
|
ELSE NULL
|
|
END
|
|
FROM (
|
|
-- Unnest the raw JSON array payload; each element is one object row.
|
|
SELECT
|
|
(raw_obj ->> 'objId')::bigint AS obj_id,
|
|
e.snapshot_date,
|
|
raw_obj
|
|
FROM domrf_raw_endpoints AS e,
|
|
jsonb_array_elements(e.payload) AS raw_obj
|
|
WHERE e.section = 'kn_api'
|
|
AND jsonb_typeof(e.payload) = 'array'
|
|
AND raw_obj ->> 'aiDescription' IS NOT NULL
|
|
) AS src
|
|
WHERE o.obj_id = src.obj_id
|
|
AND o.snapshot_date = src.snapshot_date
|
|
AND o.obj_class IS NULL
|
|
-- Only update when we can actually extract a class
|
|
AND (
|
|
src.raw_obj ->> 'aiDescription' ~* 'элит'
|
|
OR src.raw_obj ->> 'aiDescription' ~* 'бизнес'
|
|
OR src.raw_obj ->> 'aiDescription' ~* 'премиум'
|
|
OR src.raw_obj ->> 'aiDescription' ~* 'комфорт'
|
|
OR src.raw_obj ->> 'aiDescription' ~* 'стандарт|эконом'
|
|
);
|
|
|
|
-- Report result
|
|
DO $$
|
|
DECLARE
|
|
v_total bigint;
|
|
v_null bigint;
|
|
v_filled bigint;
|
|
BEGIN
|
|
SELECT COUNT(*), COUNT(*) FILTER (WHERE obj_class IS NULL),
|
|
COUNT(*) FILTER (WHERE obj_class IS NOT NULL)
|
|
INTO v_total, v_null, v_filled
|
|
FROM domrf_kn_objects;
|
|
|
|
RAISE NOTICE 'domrf_kn_objects after backfill: total=% filled=% still_null=%',
|
|
v_total, v_filled, v_null;
|
|
END$$;
|
|
|
|
COMMIT;
|