diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py index 1bbe7cd4..1c645b2b 100644 --- a/backend/app/services/scrapers/domrf_kn.py +++ b/backend/app/services/scrapers/domrf_kn.py @@ -340,7 +340,9 @@ UPSERT_OBJECT_SQL = text( ready_dt = EXCLUDED.ready_dt, site_status = EXCLUDED.site_status, escrow = EXCLUDED.escrow, - obj_class = EXCLUDED.obj_class, + obj_class = COALESCE(EXCLUDED.obj_class, domrf_kn_objects.obj_class), + wall_type = COALESCE(EXCLUDED.wall_type, domrf_kn_objects.wall_type), + energy_eff = COALESCE(EXCLUDED.energy_eff, domrf_kn_objects.energy_eff), latitude = EXCLUDED.latitude, longitude = EXCLUDED.longitude, obj_status = EXCLUDED.obj_status @@ -364,7 +366,14 @@ UPSERT_FLAT_SQL = text( price_per_m2 = EXCLUDED.price_per_m2, obj_id = EXCLUDED.obj_id, region_cd = EXCLUDED.region_cd, - obj_name = EXCLUDED.obj_name + obj_name = EXCLUDED.obj_name, + flat_number = COALESCE(EXCLUDED.flat_number, domrf_kn_flats.flat_number), + living_area = COALESCE(EXCLUDED.living_area, domrf_kn_flats.living_area), + is_studio = COALESCE(EXCLUDED.is_studio, domrf_kn_flats.is_studio), + total_area = COALESCE(EXCLUDED.total_area, domrf_kn_flats.total_area), + rooms = COALESCE(EXCLUDED.rooms, domrf_kn_flats.rooms), + floor = COALESCE(EXCLUDED.floor, domrf_kn_flats.floor), + num_floors = COALESCE(EXCLUDED.num_floors, domrf_kn_flats.num_floors) """ ) diff --git a/data/sql/108_bug22a_kn_missing_fields_audit.sql b/data/sql/108_bug22a_kn_missing_fields_audit.sql new file mode 100644 index 00000000..246f9b89 --- /dev/null +++ b/data/sql/108_bug22a_kn_missing_fields_audit.sql @@ -0,0 +1,85 @@ +-- 108_bug22a_kn_missing_fields_audit.sql +-- +-- Bug 22a audit: kn-API payload vs domrf_kn_objects / domrf_kn_flats mapping. +-- +-- FINDINGS (payload audit on obj_id=65136, all 1516 objects in kn_object_place_66, +-- snapshot 2026-05-05): +-- +-- Object-level fields (/kn/object list endpoint): +-- wallType — 0/1516 objects have key in payload → API never returns it. +-- aiDescription mentions wall material as free text but no +-- clean extraction without NLP. No backfill possible. +-- energyEff — 0/1516 objects have key in payload → API never returns it. +-- No source for backfill. +-- objClass — 0/1516 objects have key in payload → API never returns it. +-- FIXED in migration 106 (extract from aiDescription) + SF#15 +-- scraper fix. Coverage after 106: 1806/4548 filled. +-- +-- Flat-level fields (/portal-kn/api/sales/portal/table endpoint): +-- number → flat_number: mapped in _norm_flat() via _g(row, "number", "flatNumber"). +-- Confirmed present in payload for active-sale objects. +-- livingArea → living_area: mapped in _norm_flat() via _g(row, "livingArea"). +-- Confirmed present in payload for active-sale objects. +-- +-- Root cause for 0% flat_number/living_area in domrf_kn_flats: +-- UPSERT_FLAT_SQL ON CONFLICT (id, snapshot_date) DO UPDATE SET did NOT include +-- flat_number, living_area, is_studio, total_area, rooms, floor, num_floors. +-- Rows first inserted from older scrape (hash-id, no direct numeric id → NULL fields) +-- were never updated on subsequent scrapes. +-- FIX: domrf_kn.py UPSERT_FLAT_SQL now uses COALESCE for these fields. +-- Coverage will improve on next scrape run (no stored flat-level raw → no backfill). +-- +-- UPSERT_OBJECT_SQL ON CONFLICT also missed wall_type / energy_eff updates. +-- FIX: added COALESCE for wall_type and energy_eff in ON CONFLICT clause. +-- +-- All columns already exist — no DDL changes needed. This migration is a no-op +-- idempotent guard; the real fix is in backend/app/services/scrapers/domrf_kn.py. +-- +-- Idempotent: safe to re-run. + +BEGIN; + +-- Confirm columns exist (no-op if already present — guards against accidental DROP) +ALTER TABLE domrf_kn_objects + ADD COLUMN IF NOT EXISTS wall_type TEXT, + ADD COLUMN IF NOT EXISTS energy_eff TEXT; + +ALTER TABLE domrf_kn_flats + ADD COLUMN IF NOT EXISTS flat_number TEXT, + ADD COLUMN IF NOT EXISTS living_area NUMERIC(10, 2); + +-- Coverage check (informational — does not fail migration) +DO $$ +DECLARE + obj_total bigint; + obj_wall bigint; + obj_energy bigint; + obj_class_fill bigint; + flat_total bigint; + flat_number bigint; + flat_living bigint; +BEGIN + SELECT COUNT(*), + COUNT(*) FILTER (WHERE wall_type IS NOT NULL), + COUNT(*) FILTER (WHERE energy_eff IS NOT NULL), + COUNT(*) FILTER (WHERE obj_class IS NOT NULL) + INTO obj_total, obj_wall, obj_energy, obj_class_fill + FROM domrf_kn_objects WHERE region_cd = 66; + + SELECT COUNT(*), + COUNT(*) FILTER (WHERE f.flat_number IS NOT NULL), + COUNT(*) FILTER (WHERE f.living_area IS NOT NULL) + INTO flat_total, flat_number, flat_living + FROM domrf_kn_flats f + JOIN domrf_kn_objects o ON o.obj_id = f.obj_id + WHERE o.region_cd = 66; + + RAISE NOTICE + 'bug-22a coverage (region_cd=66):' + ' objects: total=% obj_class=% wall_type=% energy_eff=%' + ' | flats: total=% flat_number=% living_area=%', + obj_total, obj_class_fill, obj_wall, obj_energy, + flat_total, flat_number, flat_living; +END$$; + +COMMIT;