-- 108_bug22a_kn_missing_fields_audit.sql
--
-- Bug 22a audit: kn-API payload vs domrf_kn_objects / domrf_kn_flats mapping.
--
-- FINDINGS (payload audit on obj_id=65136, all 1516 objects in kn_object_place_66,
-- snapshot 2026-05-05):
--
--   Object-level fields (/kn/object list endpoint):
--     wallType    — 0/1516 objects have key in payload → API never returns it.
--                   aiDescription mentions wall material as free text but no
--                   clean extraction without NLP. No backfill possible.
--     energyEff   — 0/1516 objects have key in payload → API never returns it.
--                   No source for backfill.
--     objClass    — 0/1516 objects have key in payload → API never returns it.
--                   FIXED in migration 106 (extract from aiDescription) + SF#15
--                   scraper fix. Coverage after 106: 1806/4548 filled.
--
--   Flat-level fields (/portal-kn/api/sales/portal/table endpoint):
--     number      → flat_number: mapped in _norm_flat() via _g(row, "number", "flatNumber").
--                   Confirmed present in payload for active-sale objects.
--     livingArea  → living_area: mapped in _norm_flat() via _g(row, "livingArea").
--                   Confirmed present in payload for active-sale objects.
--
--   Root cause for 0% flat_number/living_area in domrf_kn_flats:
--     UPSERT_FLAT_SQL ON CONFLICT (id, snapshot_date) DO UPDATE SET did NOT include
--     flat_number, living_area, is_studio, total_area, rooms, floor, num_floors.
--     Rows first inserted from older scrape (hash-id, no direct numeric id → NULL fields)
--     were never updated on subsequent scrapes.
--     FIX: domrf_kn.py UPSERT_FLAT_SQL now uses COALESCE for these fields.
--     Coverage will improve on next scrape run (no stored flat-level raw → no backfill).
--
--   UPSERT_OBJECT_SQL ON CONFLICT also missed wall_type / energy_eff updates.
--     FIX: added COALESCE for wall_type and energy_eff in ON CONFLICT clause.
--
-- All columns already exist — no DDL changes needed. This migration is a no-op
-- idempotent guard; the real fix is in backend/app/services/scrapers/domrf_kn.py.
--
-- Idempotent: safe to re-run.

BEGIN;

-- Confirm columns exist (no-op if already present — guards against accidental DROP)
ALTER TABLE domrf_kn_objects
    ADD COLUMN IF NOT EXISTS wall_type  TEXT,
    ADD COLUMN IF NOT EXISTS energy_eff TEXT;

ALTER TABLE domrf_kn_flats
    ADD COLUMN IF NOT EXISTS flat_number TEXT,
    ADD COLUMN IF NOT EXISTS living_area NUMERIC(10, 2);

-- Coverage check (informational — does not fail migration)
DO $$
DECLARE
    obj_total      bigint;
    obj_wall       bigint;
    obj_energy     bigint;
    obj_class_fill bigint;
    flat_total     bigint;
    flat_number    bigint;
    flat_living    bigint;
BEGIN
    SELECT COUNT(*),
           COUNT(*) FILTER (WHERE wall_type IS NOT NULL),
           COUNT(*) FILTER (WHERE energy_eff IS NOT NULL),
           COUNT(*) FILTER (WHERE obj_class IS NOT NULL)
      INTO obj_total, obj_wall, obj_energy, obj_class_fill
      FROM domrf_kn_objects WHERE region_cd = 66;

    SELECT COUNT(*),
           COUNT(*) FILTER (WHERE f.flat_number IS NOT NULL),
           COUNT(*) FILTER (WHERE f.living_area IS NOT NULL)
      INTO flat_total, flat_number, flat_living
      FROM domrf_kn_flats f
      JOIN domrf_kn_objects o ON o.obj_id = f.obj_id
     WHERE o.region_cd = 66;

    RAISE NOTICE
        'bug-22a coverage (region_cd=66):'
        ' objects: total=% obj_class=% wall_type=% energy_eff=%'
        ' | flats: total=% flat_number=% living_area=%',
        obj_total, obj_class_fill, obj_wall, obj_energy,
        flat_total, flat_number, flat_living;
END$$;

COMMIT;