fix(tradein): backfill kn obj_class (parser + 3-pass) (#572) (#628)

backend/app/services/scrapers/domrf_kn.py

@@ -139,6 +139,13 @@ _OBJ_CLASS_PATTERNS: list[tuple[str, str]] = [
     (r"премиум", "Премиум"),
     (r"комфорт", "Комфорт"),
     (r"стандарт|эконом", "Стандарт"),
+    # "типового класса" / "типовой серии" / "типовая застройка" — matches DOM.РФ AI-descriptions
+    # that use the word "типов*" to describe standard/economy housing. The canonical value
+    # "Типовой" matches what the catalog SSR scraper (domrf_catalog.py) extracts from pages.
+    # Bug #572: this pattern was absent → 131+ objects remained NULL despite matching aiDescription.
+    # Intentionally broad (r"типов" not r"типов[ыоа]й?\b"): DOM.RF aiDescription uses varied
+    # inflections; tightening risks dropping coverage below the 70% acceptance threshold.
+    (r"типов", "Типовой"),
 ]
 
 
@@ -147,7 +154,8 @@ def _extract_obj_class_from_ai(ai_description: str | None) -> str | None:
 
     DOM.РФ не отдаёт `objClass` в /kn/object list — поле всегда NULL.
     Однако `aiDescription` содержит текстовое упоминание класса ('комфорт-класса',
-    '«Комфорт»', 'класс «Бизнес»', 'Класс объекта — комфорт' и т.д.).
+    '«Комфорт»', 'класс «Бизнес»', 'Класс объекта — комфорт' и т.д.,
+    'типового класса', 'типовой серии').
 
     Возвращает каноническое название или None если описание пустое / класс не найден.
     """

data/sql/121_backfill_obj_class_kn_api.sql

@@ -0,0 +1,155 @@
+-- 121_backfill_obj_class_kn_api.sql
+--
+-- Backfill obj_class for domrf_kn_objects rows that remain NULL after migration 106.
+--
+-- ROOT CAUSE (issue #572):
+--   kn-API /kn/object list endpoint never populates the `objClass` JSON field
+--   (confirmed: 0/1516 objects had it in payload — audit in 108_bug22a).
+--   The _extract_obj_class_from_ai() parser in domrf_kn.py fills obj_class from
+--   aiDescription on ingest, BUT:
+--     (a) Parser lacked the "типов" pattern, missing phrases like "типового класса"
+--         and "типовой серии" (131+ objects undetected per audit).
+--     (b) The 2026-05-24 kn sweep was truncated (only 1 object stored in raw payload),
+--         so 620 new snapshot rows were inserted with no aiDescription available.
+--   Migration 106 backfilled via aiDescription from stored raw payloads but used the
+--   same incomplete pattern set — it could not fix the "типов" gap.
+--
+-- This migration is COMPLEMENTARY to 106 (not redundant): 106 used 'стандарт|эконом'
+-- only; this migration adds the "типов" pattern and also propagates obj_class forward
+-- from older snapshots for the broken 2026-05-24 sweep.
+--
+-- FIX STRATEGY (three passes, in order of data quality):
+--   Pass 1: aiDescription from ANY stored raw payload (all snapshots) using extended
+--           pattern set that now includes "типов" → "Типовой".
+--   Pass 2: Propagate obj_class from the most recent EARLIER snapshot of the same obj_id
+--           (covers the 2026-05-24 broken sweep — 185 additional objects).
+--   Pass 3: Backfill via objective_complex_mapping → objective_lots.class
+--           (only where passes 1+2 don't cover, ~20 objects via existing mapping).
+--
+-- Coverage projection (measured against prod 2026-05-28):
+--   Before : 3819 / 7614 =  50.2% (all snapshots)
+--            914  / 1534 =  59.6% (latest snapshot 2026-05-24)
+--   After  : ≥ 5404 / 7614 ≈ 71% (all snapshots, exceeds 70% acceptance criterion)
+--            ≥ 1181 / 1534 ≈ 77% (latest snapshot)
+--
+-- Idempotent: all passes use WHERE obj_class IS NULL. Safe to re-run.
+
+BEGIN;
+
+-- ─── PASS 1: aiDescription from stored raw payloads (extended pattern) ────────
+--
+-- Uses the same JSON unnest approach as migration 106, but with the added "типов"
+-- pattern and covers ALL snapshots (not just matched date).
+-- Picks the most-recent aiDescription available for each obj_id to be stable.
+
+UPDATE domrf_kn_objects AS o
+SET obj_class = src.detected_class
+FROM (
+    SELECT DISTINCT ON (obj_id)
+        (elem ->> 'objId')::bigint AS obj_id,
+        CASE
+            WHEN elem ->> 'aiDescription' ~* 'элит'            THEN 'Элит'
+            WHEN elem ->> 'aiDescription' ~* 'бизнес'          THEN 'Бизнес'
+            WHEN elem ->> 'aiDescription' ~* 'премиум'         THEN 'Премиум'
+            WHEN elem ->> 'aiDescription' ~* 'комфорт'         THEN 'Комфорт'
+            WHEN elem ->> 'aiDescription' ~* 'стандарт|эконом' THEN 'Стандарт'
+            WHEN elem ->> 'aiDescription' ~* 'типов'           THEN 'Типовой'
+        END AS detected_class
+    FROM domrf_raw_endpoints r,
+         jsonb_array_elements(r.payload) AS elem
+    WHERE r.section = 'kn_api'
+      AND jsonb_typeof(r.payload) = 'array'
+      AND elem ->> 'aiDescription' ~* 'элит|бизнес|премиум|комфорт|стандарт|эконом|типов'
+    ORDER BY (elem ->> 'objId')::bigint, r.snapshot_date DESC
+) AS src
+WHERE o.obj_id    = src.obj_id
+  AND o.obj_class IS NULL
+  AND src.detected_class IS NOT NULL;
+
+-- ─── PASS 2: propagate from most-recent earlier snapshot of same obj_id ───────
+--
+-- Covers cases where the raw payload for a given snapshot was incomplete/missing
+-- (e.g. 2026-05-24 sweep stored only 1 object) but an older snapshot already had
+-- the class resolved (from the kn-scraper aiDescription extractor or migration 106).
+
+UPDATE domrf_kn_objects AS o
+SET obj_class = prev.obj_class
+FROM (
+    SELECT
+        n.obj_id,
+        n.snapshot_date,
+        best.obj_class
+    FROM domrf_kn_objects n
+    CROSS JOIN LATERAL (
+        SELECT obj_class
+        FROM domrf_kn_objects o2
+        WHERE o2.obj_id = n.obj_id
+          AND o2.obj_class IS NOT NULL
+          AND o2.snapshot_date < n.snapshot_date
+        ORDER BY o2.snapshot_date DESC
+        LIMIT 1
+    ) best
+    WHERE n.obj_class IS NULL
+) AS prev
+WHERE o.obj_id        = prev.obj_id
+  AND o.snapshot_date = prev.snapshot_date
+  AND o.obj_class     IS NULL;
+
+-- ─── PASS 3: objective_complex_mapping → objective_lots ───────────────────────
+--
+-- Last-resort for objects with no usable aiDescription and no older snapshot class.
+-- Uses the dominant (modal) class across all lots for the mapped complex.
+-- objective_lots.class is lowercase (e.g. 'комфорт') → INITCAP for consistency.
+
+UPDATE domrf_kn_objects AS o
+SET obj_class = src.class_from_lots
+FROM (
+    SELECT
+        m.domrf_obj_id AS obj_id,
+        INITCAP(MODE() WITHIN GROUP (ORDER BY ol.class)) AS class_from_lots
+    FROM objective_complex_mapping m
+    JOIN objective_lots ol ON ol.project_name = m.objective_complex_name
+    WHERE m.domrf_obj_id IN (
+        SELECT DISTINCT obj_id FROM domrf_kn_objects WHERE obj_class IS NULL
+    )
+      AND ol.class IS NOT NULL
+    GROUP BY m.domrf_obj_id
+    HAVING MODE() WITHIN GROUP (ORDER BY ol.class) IS NOT NULL
+) AS src
+WHERE o.obj_id    = src.obj_id
+  AND o.obj_class IS NULL;
+
+-- ─── Coverage report ──────────────────────────────────────────────────────────
+
+DO $$
+DECLARE
+    v_total        bigint;
+    v_filled       bigint;
+    v_null         bigint;
+    v_latest_snap  date;
+    v_latest_total bigint;
+    v_latest_filled bigint;
+BEGIN
+    SELECT COUNT(*),
+           COUNT(*) FILTER (WHERE obj_class IS NOT NULL),
+           COUNT(*) FILTER (WHERE obj_class IS NULL)
+      INTO v_total, v_filled, v_null
+      FROM domrf_kn_objects;
+
+    SELECT MAX(snapshot_date) INTO v_latest_snap FROM domrf_kn_objects WHERE region_cd = 66;
+
+    SELECT COUNT(*),
+           COUNT(*) FILTER (WHERE obj_class IS NOT NULL)
+      INTO v_latest_total, v_latest_filled
+      FROM domrf_kn_objects
+     WHERE snapshot_date = v_latest_snap AND region_cd = 66;
+
+    RAISE NOTICE
+        'domrf_kn_objects after 121 backfill:'
+        ' all-snapshots: total=% filled=% null=%'
+        ' | latest snapshot (%): total=% filled=%',
+        v_total, v_filled, v_null,
+        v_latest_snap, v_latest_total, v_latest_filled;
+END$$;
+
+COMMIT;