backend/app/services/scrapers/domrf_kn.py

@@ -340,7 +340,9 @@ UPSERT_OBJECT_SQL = text(
         ready_dt = EXCLUDED.ready_dt,
         site_status = EXCLUDED.site_status,
         escrow = EXCLUDED.escrow,
-        obj_class = EXCLUDED.obj_class,
+        obj_class = COALESCE(EXCLUDED.obj_class, domrf_kn_objects.obj_class),
+        wall_type = COALESCE(EXCLUDED.wall_type, domrf_kn_objects.wall_type),
+        energy_eff = COALESCE(EXCLUDED.energy_eff, domrf_kn_objects.energy_eff),
         latitude = EXCLUDED.latitude,
         longitude = EXCLUDED.longitude,
         obj_status = EXCLUDED.obj_status
@@ -364,7 +366,14 @@ UPSERT_FLAT_SQL = text(
         price_per_m2 = EXCLUDED.price_per_m2,
         obj_id = EXCLUDED.obj_id,
         region_cd = EXCLUDED.region_cd,
-        obj_name = EXCLUDED.obj_name
+        obj_name = EXCLUDED.obj_name,
+        flat_number = COALESCE(EXCLUDED.flat_number, domrf_kn_flats.flat_number),
+        living_area = COALESCE(EXCLUDED.living_area, domrf_kn_flats.living_area),
+        is_studio = COALESCE(EXCLUDED.is_studio, domrf_kn_flats.is_studio),
+        total_area = COALESCE(EXCLUDED.total_area, domrf_kn_flats.total_area),
+        rooms = COALESCE(EXCLUDED.rooms, domrf_kn_flats.rooms),
+        floor = COALESCE(EXCLUDED.floor, domrf_kn_flats.floor),
+        num_floors = COALESCE(EXCLUDED.num_floors, domrf_kn_flats.num_floors)
     """
 )
 

data/sql/108_bug22a_kn_missing_fields_audit.sql

@@ -0,0 +1,85 @@
+-- 108_bug22a_kn_missing_fields_audit.sql
+--
+-- Bug 22a audit: kn-API payload vs domrf_kn_objects / domrf_kn_flats mapping.
+--
+-- FINDINGS (payload audit on obj_id=65136, all 1516 objects in kn_object_place_66,
+-- snapshot 2026-05-05):
+--
+--   Object-level fields (/kn/object list endpoint):
+--     wallType    — 0/1516 objects have key in payload → API never returns it.
+--                   aiDescription mentions wall material as free text but no
+--                   clean extraction without NLP. No backfill possible.
+--     energyEff   — 0/1516 objects have key in payload → API never returns it.
+--                   No source for backfill.
+--     objClass    — 0/1516 objects have key in payload → API never returns it.
+--                   FIXED in migration 106 (extract from aiDescription) + SF#15
+--                   scraper fix. Coverage after 106: 1806/4548 filled.
+--
+--   Flat-level fields (/portal-kn/api/sales/portal/table endpoint):
+--     number      → flat_number: mapped in _norm_flat() via _g(row, "number", "flatNumber").
+--                   Confirmed present in payload for active-sale objects.
+--     livingArea  → living_area: mapped in _norm_flat() via _g(row, "livingArea").
+--                   Confirmed present in payload for active-sale objects.
+--
+--   Root cause for 0% flat_number/living_area in domrf_kn_flats:
+--     UPSERT_FLAT_SQL ON CONFLICT (id, snapshot_date) DO UPDATE SET did NOT include
+--     flat_number, living_area, is_studio, total_area, rooms, floor, num_floors.
+--     Rows first inserted from older scrape (hash-id, no direct numeric id → NULL fields)
+--     were never updated on subsequent scrapes.
+--     FIX: domrf_kn.py UPSERT_FLAT_SQL now uses COALESCE for these fields.
+--     Coverage will improve on next scrape run (no stored flat-level raw → no backfill).
+--
+--   UPSERT_OBJECT_SQL ON CONFLICT also missed wall_type / energy_eff updates.
+--     FIX: added COALESCE for wall_type and energy_eff in ON CONFLICT clause.
+--
+-- All columns already exist — no DDL changes needed. This migration is a no-op
+-- idempotent guard; the real fix is in backend/app/services/scrapers/domrf_kn.py.
+--
+-- Idempotent: safe to re-run.
+
+BEGIN;
+
+-- Confirm columns exist (no-op if already present — guards against accidental DROP)
+ALTER TABLE domrf_kn_objects
+    ADD COLUMN IF NOT EXISTS wall_type  TEXT,
+    ADD COLUMN IF NOT EXISTS energy_eff TEXT;
+
+ALTER TABLE domrf_kn_flats
+    ADD COLUMN IF NOT EXISTS flat_number TEXT,
+    ADD COLUMN IF NOT EXISTS living_area NUMERIC(10, 2);
+
+-- Coverage check (informational — does not fail migration)
+DO $$
+DECLARE
+    obj_total      bigint;
+    obj_wall       bigint;
+    obj_energy     bigint;
+    obj_class_fill bigint;
+    flat_total     bigint;
+    flat_number    bigint;
+    flat_living    bigint;
+BEGIN
+    SELECT COUNT(*),
+           COUNT(*) FILTER (WHERE wall_type IS NOT NULL),
+           COUNT(*) FILTER (WHERE energy_eff IS NOT NULL),
+           COUNT(*) FILTER (WHERE obj_class IS NOT NULL)
+      INTO obj_total, obj_wall, obj_energy, obj_class_fill
+      FROM domrf_kn_objects WHERE region_cd = 66;
+
+    SELECT COUNT(*),
+           COUNT(*) FILTER (WHERE f.flat_number IS NOT NULL),
+           COUNT(*) FILTER (WHERE f.living_area IS NOT NULL)
+      INTO flat_total, flat_number, flat_living
+      FROM domrf_kn_flats f
+      JOIN domrf_kn_objects o ON o.obj_id = f.obj_id
+     WHERE o.region_cd = 66;
+
+    RAISE NOTICE
+        'bug-22a coverage (region_cd=66):'
+        ' objects: total=% obj_class=% wall_type=% energy_eff=%'
+        ' | flats: total=% flat_number=% living_area=%',
+        obj_total, obj_class_fill, obj_wall, obj_energy,
+        flat_total, flat_number, flat_living;
+END$$;
+
+COMMIT;