backend/app/services/scrapers/domrf_kn.py

@@ -575,7 +575,10 @@ UPSERT_FLAT_SQL = text(
     ON CONFLICT (id, snapshot_date) DO UPDATE SET
         status = EXCLUDED.status,
         price_rub = EXCLUDED.price_rub,
-        price_per_m2 = EXCLUDED.price_per_m2,
+        -- COALESCE: keep a backfilled / sale_graph-derived price_per_m2 when the
+        -- scraper produces NULL (NULL-status historical flats have no price_rub).
+        -- Direct scraper value always wins when non-NULL.
+        price_per_m2 = COALESCE(EXCLUDED.price_per_m2, domrf_kn_flats.price_per_m2),
         obj_id = EXCLUDED.obj_id,
         region_cd = EXCLUDED.region_cd,
         obj_name = EXCLUDED.obj_name,

data/sql/120_backfill_kn_price_per_m2.sql

@@ -0,0 +1,76 @@
+-- 120_backfill_kn_price_per_m2.sql
+-- Context: Issue #573 — domrf_kn_flats.price_per_m2 is populated in only ~3.7%
+--   of rows (status='free'/'sold'/'booked'). The 756 977 NULL-status rows imported
+--   from historical DOM.RF snapshots carry total_area but no price_rub, so the
+--   scraper's price_rub/total_area derivation never fires for them.
+--   This leaves 5/8 EKB districts with 0% price_per_m2 coverage and NULL district
+--   medians, blocking the Limit_District_Median_Partial analytics limitation.
+--
+-- Strategy (three-pass, most-precise source first):
+--   Pass 1 — direct arithmetic: price_rub / total_area where both columns present
+--             and price_per_m2 is still NULL or zero.  Covers any rows the scraper
+--             missed (idempotent sanity net).
+--   Pass 2 — sale_graph join: use domrf_kn_sale_graph.price_avg (type='apartments',
+--             latest snapshot) as a proxy for the object-level avg ₽/м².  Applied
+--             only to rows where Pass 1 left price_per_m2 NULL and the object has
+--             a sale_graph row.
+--   Pass 3 — peer median: for objects without sale_graph, derive price_per_m2 from
+--             the PERCENTILE_CONT(0.5) of free/sold/booked flats in the same obj_id.
+--
+-- Idempotent: WHERE price_per_m2 IS NULL guard prevents re-overwriting correct values.
+-- Dependencies: domrf_kn_flats, domrf_kn_sale_graph (migration 50_schema_kn_extensions.sql).
+-- Apply order: after 99_nspd_entities_denorm.sql.
+-- Estimated runtime on prod: < 30 s (785 K rows, index on obj_id exists).
+
+BEGIN;
+
+-- ── Pass 1: direct price_rub / total_area ────────────────────────────────────
+UPDATE public.domrf_kn_flats
+   SET price_per_m2 = ROUND(price_rub / total_area, 2)
+ WHERE price_rub IS NOT NULL
+   AND total_area IS NOT NULL
+   AND total_area > 0
+   AND (price_per_m2 IS NULL OR price_per_m2 = 0);
+
+-- ── Pass 2: sale_graph proxy (latest snapshot, type='apartments') ─────────────
+-- domrf_kn_sale_graph.price_avg is the average contract ₽/м² reported by DOM.RF
+-- for the object in a given month.  We use the most recent snapshot's average
+-- across all apartment sub-rows as a reasonable object-level proxy.
+WITH sg_latest AS (
+    SELECT obj_id,
+           AVG(price_avg) AS avg_ppm2
+      FROM public.domrf_kn_sale_graph
+     WHERE type = 'apartments'
+       AND price_avg > 50000
+       AND snapshot_date = (
+               SELECT MAX(snapshot_date)
+                 FROM public.domrf_kn_sale_graph
+                WHERE type = 'apartments'
+       )
+     GROUP BY obj_id
+)
+UPDATE public.domrf_kn_flats f
+   SET price_per_m2 = ROUND(sg.avg_ppm2, 2)
+  FROM sg_latest sg
+ WHERE f.obj_id = sg.obj_id
+   AND f.price_per_m2 IS NULL;
+
+-- ── Pass 3: peer median from free/sold/booked flats in same obj ───────────────
+-- Fallback for objects absent from sale_graph: compute the object-level median
+-- from the priced (status IS NOT NULL) flats in the same building.
+WITH peer AS (
+    SELECT obj_id,
+           PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY price_per_m2) AS median_ppm2
+      FROM public.domrf_kn_flats
+     WHERE status IS NOT NULL
+       AND price_per_m2 IS NOT NULL
+       AND price_per_m2 > 0
+     GROUP BY obj_id
+)
+UPDATE public.domrf_kn_flats f
+   SET price_per_m2 = ROUND(p.median_ppm2::numeric, 2)
+  FROM peer p
+ WHERE f.obj_id = p.obj_id
+   AND f.price_per_m2 IS NULL;
+
+COMMIT;