fix(domrf): backfill kn obj_class via aiDescription + older snapshots (#572)
kn-API /kn/object endpoint never populates objClass (0/1516 objects confirmed
in bug-22a audit). Parser _extract_obj_class_from_ai() worked but lacked the
"типов" pattern, missing "типового класса"/"типовой серии" in 131+ objects.
The 2026-05-24 sweep was also truncated (1 object in raw payload) leaving 620
new rows with no aiDescription to extract from.
Fix:
- Add (r"типов", "Типовой") to _OBJ_CLASS_PATTERNS in domrf_kn.py; aligns with
the "Типовой" value that domrf_catalog.py (SSR scraper) already sets from pages.
- Migration 120: three-pass backfill covering all snapshots:
Pass 1: aiDescription from all stored raw payloads with extended pattern
Pass 2: propagate obj_class forward from most recent older snapshot (covers
broken 2026-05-24 sweep — 185 objects)
Pass 3: objective_complex_mapping → objective_lots modal class (~14 objects)
Projects 50.2% → 71.2% coverage (3819→5418/7614 rows); meets ≥70% acceptance.
This commit is contained in:
parent
6553e5830f
commit
dfae54384e
2 changed files with 162 additions and 1 deletions
|
|
@ -139,6 +139,11 @@ _OBJ_CLASS_PATTERNS: list[tuple[str, str]] = [
|
|||
(r"премиум", "Премиум"),
|
||||
(r"комфорт", "Комфорт"),
|
||||
(r"стандарт|эконом", "Стандарт"),
|
||||
# "типового класса" / "типовой серии" / "типовая застройка" — matches DOM.РФ AI-descriptions
|
||||
# that use the word "типов*" to describe standard/economy housing. The canonical value
|
||||
# "Типовой" matches what the catalog SSR scraper (domrf_catalog.py) extracts from pages.
|
||||
# Bug #572: this pattern was absent → 131+ objects remained NULL despite matching aiDescription.
|
||||
(r"типов", "Типовой"),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -147,7 +152,8 @@ def _extract_obj_class_from_ai(ai_description: str | None) -> str | None:
|
|||
|
||||
DOM.РФ не отдаёт `objClass` в /kn/object list — поле всегда NULL.
|
||||
Однако `aiDescription` содержит текстовое упоминание класса ('комфорт-класса',
|
||||
'«Комфорт»', 'класс «Бизнес»', 'Класс объекта — комфорт' и т.д.).
|
||||
'«Комфорт»', 'класс «Бизнес»', 'Класс объекта — комфорт' и т.д.,
|
||||
'типового класса', 'типовой серии').
|
||||
|
||||
Возвращает каноническое название или None если описание пустое / класс не найден.
|
||||
"""
|
||||
|
|
|
|||
155
data/sql/120_backfill_obj_class_kn_api.sql
Normal file
155
data/sql/120_backfill_obj_class_kn_api.sql
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
-- 120_backfill_obj_class_kn_api.sql
|
||||
--
|
||||
-- Backfill obj_class for domrf_kn_objects rows that remain NULL after migration 106.
|
||||
--
|
||||
-- ROOT CAUSE (issue #572):
|
||||
-- kn-API /kn/object list endpoint never populates the `objClass` JSON field
|
||||
-- (confirmed: 0/1516 objects had it in payload — audit in 108_bug22a).
|
||||
-- The _extract_obj_class_from_ai() parser in domrf_kn.py fills obj_class from
|
||||
-- aiDescription on ingest, BUT:
|
||||
-- (a) Parser lacked the "типов" pattern, missing phrases like "типового класса"
|
||||
-- and "типовой серии" (131+ objects undetected per audit).
|
||||
-- (b) The 2026-05-24 kn sweep was truncated (only 1 object stored in raw payload),
|
||||
-- so 620 new snapshot rows were inserted with no aiDescription available.
|
||||
-- Migration 106 backfilled via aiDescription from stored raw payloads but used the
|
||||
-- same incomplete pattern set — it could not fix the "типов" gap.
|
||||
--
|
||||
-- This migration is COMPLEMENTARY to 106 (not redundant): 106 used 'стандарт|эконом'
|
||||
-- only; this migration adds the "типов" pattern and also propagates obj_class forward
|
||||
-- from older snapshots for the broken 2026-05-24 sweep.
|
||||
--
|
||||
-- FIX STRATEGY (three passes, in order of data quality):
|
||||
-- Pass 1: aiDescription from ANY stored raw payload (all snapshots) using extended
|
||||
-- pattern set that now includes "типов" → "Типовой".
|
||||
-- Pass 2: Propagate obj_class from the most recent EARLIER snapshot of the same obj_id
|
||||
-- (covers the 2026-05-24 broken sweep — 185 additional objects).
|
||||
-- Pass 3: Backfill via objective_complex_mapping → objective_lots.class
|
||||
-- (only where passes 1+2 don't cover, ~20 objects via existing mapping).
|
||||
--
|
||||
-- Coverage projection (measured against prod 2026-05-28):
|
||||
-- Before : 3819 / 7614 = 50.2% (all snapshots)
|
||||
-- 914 / 1534 = 59.6% (latest snapshot 2026-05-24)
|
||||
-- After : ≥ 5404 / 7614 ≈ 71% (all snapshots, exceeds 70% acceptance criterion)
|
||||
-- ≥ 1181 / 1534 ≈ 77% (latest snapshot)
|
||||
--
|
||||
-- Idempotent: all passes use WHERE obj_class IS NULL. Safe to re-run.
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ─── PASS 1: aiDescription from stored raw payloads (extended pattern) ────────
|
||||
--
|
||||
-- Uses the same JSON unnest approach as migration 106, but with the added "типов"
|
||||
-- pattern and covers ALL snapshots (not just matched date).
|
||||
-- Picks the most-recent aiDescription available for each obj_id to be stable.
|
||||
|
||||
UPDATE domrf_kn_objects AS o
|
||||
SET obj_class = src.detected_class
|
||||
FROM (
|
||||
SELECT DISTINCT ON (obj_id)
|
||||
(elem ->> 'objId')::bigint AS obj_id,
|
||||
CASE
|
||||
WHEN elem ->> 'aiDescription' ~* 'элит' THEN 'Элит'
|
||||
WHEN elem ->> 'aiDescription' ~* 'бизнес' THEN 'Бизнес'
|
||||
WHEN elem ->> 'aiDescription' ~* 'премиум' THEN 'Премиум'
|
||||
WHEN elem ->> 'aiDescription' ~* 'комфорт' THEN 'Комфорт'
|
||||
WHEN elem ->> 'aiDescription' ~* 'стандарт|эконом' THEN 'Стандарт'
|
||||
WHEN elem ->> 'aiDescription' ~* 'типов' THEN 'Типовой'
|
||||
END AS detected_class
|
||||
FROM domrf_raw_endpoints r,
|
||||
jsonb_array_elements(r.payload) AS elem
|
||||
WHERE r.section = 'kn_api'
|
||||
AND jsonb_typeof(r.payload) = 'array'
|
||||
AND elem ->> 'aiDescription' ~* 'элит|бизнес|премиум|комфорт|стандарт|эконом|типов'
|
||||
ORDER BY (elem ->> 'objId')::bigint, r.snapshot_date DESC
|
||||
) AS src
|
||||
WHERE o.obj_id = src.obj_id
|
||||
AND o.obj_class IS NULL
|
||||
AND src.detected_class IS NOT NULL;
|
||||
|
||||
-- ─── PASS 2: propagate from most-recent earlier snapshot of same obj_id ───────
|
||||
--
|
||||
-- Covers cases where the raw payload for a given snapshot was incomplete/missing
|
||||
-- (e.g. 2026-05-24 sweep stored only 1 object) but an older snapshot already had
|
||||
-- the class resolved (from the kn-scraper aiDescription extractor or migration 106).
|
||||
|
||||
UPDATE domrf_kn_objects AS o
|
||||
SET obj_class = prev.obj_class
|
||||
FROM (
|
||||
SELECT DISTINCT ON (n.obj_id, n.snapshot_date)
|
||||
n.obj_id,
|
||||
n.snapshot_date,
|
||||
best.obj_class
|
||||
FROM domrf_kn_objects n
|
||||
CROSS JOIN LATERAL (
|
||||
SELECT obj_class
|
||||
FROM domrf_kn_objects o2
|
||||
WHERE o2.obj_id = n.obj_id
|
||||
AND o2.obj_class IS NOT NULL
|
||||
AND o2.snapshot_date < n.snapshot_date
|
||||
ORDER BY o2.snapshot_date DESC
|
||||
LIMIT 1
|
||||
) best
|
||||
WHERE n.obj_class IS NULL
|
||||
) AS prev
|
||||
WHERE o.obj_id = prev.obj_id
|
||||
AND o.snapshot_date = prev.snapshot_date
|
||||
AND o.obj_class IS NULL;
|
||||
|
||||
-- ─── PASS 3: objective_complex_mapping → objective_lots ───────────────────────
|
||||
--
|
||||
-- Last-resort for objects with no usable aiDescription and no older snapshot class.
|
||||
-- Uses the dominant (modal) class across all lots for the mapped complex.
|
||||
-- objective_lots.class is lowercase (e.g. 'комфорт') → INITCAP for consistency.
|
||||
|
||||
UPDATE domrf_kn_objects AS o
|
||||
SET obj_class = src.class_from_lots
|
||||
FROM (
|
||||
SELECT
|
||||
m.domrf_obj_id AS obj_id,
|
||||
INITCAP(MODE() WITHIN GROUP (ORDER BY ol.class)) AS class_from_lots
|
||||
FROM objective_complex_mapping m
|
||||
JOIN objective_lots ol ON ol.project_name = m.objective_complex_name
|
||||
WHERE m.domrf_obj_id IN (
|
||||
SELECT DISTINCT obj_id FROM domrf_kn_objects WHERE obj_class IS NULL
|
||||
)
|
||||
AND ol.class IS NOT NULL
|
||||
GROUP BY m.domrf_obj_id
|
||||
HAVING MODE() WITHIN GROUP (ORDER BY ol.class) IS NOT NULL
|
||||
) AS src
|
||||
WHERE o.obj_id = src.obj_id
|
||||
AND o.obj_class IS NULL;
|
||||
|
||||
-- ─── Coverage report ──────────────────────────────────────────────────────────
|
||||
|
||||
DO $$
|
||||
DECLARE
|
||||
v_total bigint;
|
||||
v_filled bigint;
|
||||
v_null bigint;
|
||||
v_latest_snap date;
|
||||
v_latest_total bigint;
|
||||
v_latest_filled bigint;
|
||||
BEGIN
|
||||
SELECT COUNT(*),
|
||||
COUNT(*) FILTER (WHERE obj_class IS NOT NULL),
|
||||
COUNT(*) FILTER (WHERE obj_class IS NULL)
|
||||
INTO v_total, v_filled, v_null
|
||||
FROM domrf_kn_objects;
|
||||
|
||||
SELECT MAX(snapshot_date) INTO v_latest_snap FROM domrf_kn_objects WHERE region_cd = 66;
|
||||
|
||||
SELECT COUNT(*),
|
||||
COUNT(*) FILTER (WHERE obj_class IS NOT NULL)
|
||||
INTO v_latest_total, v_latest_filled
|
||||
FROM domrf_kn_objects
|
||||
WHERE snapshot_date = v_latest_snap AND region_cd = 66;
|
||||
|
||||
RAISE NOTICE
|
||||
'domrf_kn_objects after 120 backfill:'
|
||||
' all-snapshots: total=% filled=% null=%'
|
||||
' | latest snapshot (%): total=% filled=%',
|
||||
v_total, v_filled, v_null,
|
||||
v_latest_snap, v_latest_total, v_latest_filled;
|
||||
END$$;
|
||||
|
||||
COMMIT;
|
||||
Loading…
Add table
Reference in a new issue