From 4e4762f2342af309fa643911605bd2512e225a60 Mon Sep 17 00:00:00 2001 From: Light1YT Date: Thu, 28 May 2026 15:04:32 +0000 Subject: [PATCH] fix(tradein): backfill kn obj_class (parser + 3-pass) (#572) (#628) --- backend/app/services/scrapers/domrf_kn.py | 10 +- data/sql/121_backfill_obj_class_kn_api.sql | 155 +++++++++++++++++++++ 2 files changed, 164 insertions(+), 1 deletion(-) create mode 100644 data/sql/121_backfill_obj_class_kn_api.sql diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py index ff3fdc82..6e26498e 100644 --- a/backend/app/services/scrapers/domrf_kn.py +++ b/backend/app/services/scrapers/domrf_kn.py @@ -139,6 +139,13 @@ _OBJ_CLASS_PATTERNS: list[tuple[str, str]] = [ (r"премиум", "Премиум"), (r"комфорт", "Комфорт"), (r"стандарт|эконом", "Стандарт"), + # "типового класса" / "типовой серии" / "типовая застройка" — matches DOM.РФ AI-descriptions + # that use the word "типов*" to describe standard/economy housing. The canonical value + # "Типовой" matches what the catalog SSR scraper (domrf_catalog.py) extracts from pages. + # Bug #572: this pattern was absent → 131+ objects remained NULL despite matching aiDescription. + # Intentionally broad (r"типов" not r"типов[ыоа]й?\b"): DOM.RF aiDescription uses varied + # inflections; tightening risks dropping coverage below the 70% acceptance threshold. + (r"типов", "Типовой"), ] @@ -147,7 +154,8 @@ def _extract_obj_class_from_ai(ai_description: str | None) -> str | None: DOM.РФ не отдаёт `objClass` в /kn/object list — поле всегда NULL. Однако `aiDescription` содержит текстовое упоминание класса ('комфорт-класса', - '«Комфорт»', 'класс «Бизнес»', 'Класс объекта — комфорт' и т.д.). + '«Комфорт»', 'класс «Бизнес»', 'Класс объекта — комфорт' и т.д., + 'типового класса', 'типовой серии'). Возвращает каноническое название или None если описание пустое / класс не найден. """ diff --git a/data/sql/121_backfill_obj_class_kn_api.sql b/data/sql/121_backfill_obj_class_kn_api.sql new file mode 100644 index 00000000..94b659ea --- /dev/null +++ b/data/sql/121_backfill_obj_class_kn_api.sql @@ -0,0 +1,155 @@ +-- 121_backfill_obj_class_kn_api.sql +-- +-- Backfill obj_class for domrf_kn_objects rows that remain NULL after migration 106. +-- +-- ROOT CAUSE (issue #572): +-- kn-API /kn/object list endpoint never populates the `objClass` JSON field +-- (confirmed: 0/1516 objects had it in payload — audit in 108_bug22a). +-- The _extract_obj_class_from_ai() parser in domrf_kn.py fills obj_class from +-- aiDescription on ingest, BUT: +-- (a) Parser lacked the "типов" pattern, missing phrases like "типового класса" +-- and "типовой серии" (131+ objects undetected per audit). +-- (b) The 2026-05-24 kn sweep was truncated (only 1 object stored in raw payload), +-- so 620 new snapshot rows were inserted with no aiDescription available. +-- Migration 106 backfilled via aiDescription from stored raw payloads but used the +-- same incomplete pattern set — it could not fix the "типов" gap. +-- +-- This migration is COMPLEMENTARY to 106 (not redundant): 106 used 'стандарт|эконом' +-- only; this migration adds the "типов" pattern and also propagates obj_class forward +-- from older snapshots for the broken 2026-05-24 sweep. +-- +-- FIX STRATEGY (three passes, in order of data quality): +-- Pass 1: aiDescription from ANY stored raw payload (all snapshots) using extended +-- pattern set that now includes "типов" → "Типовой". +-- Pass 2: Propagate obj_class from the most recent EARLIER snapshot of the same obj_id +-- (covers the 2026-05-24 broken sweep — 185 additional objects). +-- Pass 3: Backfill via objective_complex_mapping → objective_lots.class +-- (only where passes 1+2 don't cover, ~20 objects via existing mapping). +-- +-- Coverage projection (measured against prod 2026-05-28): +-- Before : 3819 / 7614 = 50.2% (all snapshots) +-- 914 / 1534 = 59.6% (latest snapshot 2026-05-24) +-- After : ≥ 5404 / 7614 ≈ 71% (all snapshots, exceeds 70% acceptance criterion) +-- ≥ 1181 / 1534 ≈ 77% (latest snapshot) +-- +-- Idempotent: all passes use WHERE obj_class IS NULL. Safe to re-run. + +BEGIN; + +-- ─── PASS 1: aiDescription from stored raw payloads (extended pattern) ──────── +-- +-- Uses the same JSON unnest approach as migration 106, but with the added "типов" +-- pattern and covers ALL snapshots (not just matched date). +-- Picks the most-recent aiDescription available for each obj_id to be stable. + +UPDATE domrf_kn_objects AS o +SET obj_class = src.detected_class +FROM ( + SELECT DISTINCT ON (obj_id) + (elem ->> 'objId')::bigint AS obj_id, + CASE + WHEN elem ->> 'aiDescription' ~* 'элит' THEN 'Элит' + WHEN elem ->> 'aiDescription' ~* 'бизнес' THEN 'Бизнес' + WHEN elem ->> 'aiDescription' ~* 'премиум' THEN 'Премиум' + WHEN elem ->> 'aiDescription' ~* 'комфорт' THEN 'Комфорт' + WHEN elem ->> 'aiDescription' ~* 'стандарт|эконом' THEN 'Стандарт' + WHEN elem ->> 'aiDescription' ~* 'типов' THEN 'Типовой' + END AS detected_class + FROM domrf_raw_endpoints r, + jsonb_array_elements(r.payload) AS elem + WHERE r.section = 'kn_api' + AND jsonb_typeof(r.payload) = 'array' + AND elem ->> 'aiDescription' ~* 'элит|бизнес|премиум|комфорт|стандарт|эконом|типов' + ORDER BY (elem ->> 'objId')::bigint, r.snapshot_date DESC +) AS src +WHERE o.obj_id = src.obj_id + AND o.obj_class IS NULL + AND src.detected_class IS NOT NULL; + +-- ─── PASS 2: propagate from most-recent earlier snapshot of same obj_id ─────── +-- +-- Covers cases where the raw payload for a given snapshot was incomplete/missing +-- (e.g. 2026-05-24 sweep stored only 1 object) but an older snapshot already had +-- the class resolved (from the kn-scraper aiDescription extractor or migration 106). + +UPDATE domrf_kn_objects AS o +SET obj_class = prev.obj_class +FROM ( + SELECT + n.obj_id, + n.snapshot_date, + best.obj_class + FROM domrf_kn_objects n + CROSS JOIN LATERAL ( + SELECT obj_class + FROM domrf_kn_objects o2 + WHERE o2.obj_id = n.obj_id + AND o2.obj_class IS NOT NULL + AND o2.snapshot_date < n.snapshot_date + ORDER BY o2.snapshot_date DESC + LIMIT 1 + ) best + WHERE n.obj_class IS NULL +) AS prev +WHERE o.obj_id = prev.obj_id + AND o.snapshot_date = prev.snapshot_date + AND o.obj_class IS NULL; + +-- ─── PASS 3: objective_complex_mapping → objective_lots ─────────────────────── +-- +-- Last-resort for objects with no usable aiDescription and no older snapshot class. +-- Uses the dominant (modal) class across all lots for the mapped complex. +-- objective_lots.class is lowercase (e.g. 'комфорт') → INITCAP for consistency. + +UPDATE domrf_kn_objects AS o +SET obj_class = src.class_from_lots +FROM ( + SELECT + m.domrf_obj_id AS obj_id, + INITCAP(MODE() WITHIN GROUP (ORDER BY ol.class)) AS class_from_lots + FROM objective_complex_mapping m + JOIN objective_lots ol ON ol.project_name = m.objective_complex_name + WHERE m.domrf_obj_id IN ( + SELECT DISTINCT obj_id FROM domrf_kn_objects WHERE obj_class IS NULL + ) + AND ol.class IS NOT NULL + GROUP BY m.domrf_obj_id + HAVING MODE() WITHIN GROUP (ORDER BY ol.class) IS NOT NULL +) AS src +WHERE o.obj_id = src.obj_id + AND o.obj_class IS NULL; + +-- ─── Coverage report ────────────────────────────────────────────────────────── + +DO $$ +DECLARE + v_total bigint; + v_filled bigint; + v_null bigint; + v_latest_snap date; + v_latest_total bigint; + v_latest_filled bigint; +BEGIN + SELECT COUNT(*), + COUNT(*) FILTER (WHERE obj_class IS NOT NULL), + COUNT(*) FILTER (WHERE obj_class IS NULL) + INTO v_total, v_filled, v_null + FROM domrf_kn_objects; + + SELECT MAX(snapshot_date) INTO v_latest_snap FROM domrf_kn_objects WHERE region_cd = 66; + + SELECT COUNT(*), + COUNT(*) FILTER (WHERE obj_class IS NOT NULL) + INTO v_latest_total, v_latest_filled + FROM domrf_kn_objects + WHERE snapshot_date = v_latest_snap AND region_cd = 66; + + RAISE NOTICE + 'domrf_kn_objects after 121 backfill:' + ' all-snapshots: total=% filled=% null=%' + ' | latest snapshot (%): total=% filled=%', + v_total, v_filled, v_null, + v_latest_snap, v_latest_total, v_latest_filled; +END$$; + +COMMIT;