diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py index 765a93dc..ff3fdc82 100644 --- a/backend/app/services/scrapers/domrf_kn.py +++ b/backend/app/services/scrapers/domrf_kn.py @@ -575,7 +575,10 @@ UPSERT_FLAT_SQL = text( ON CONFLICT (id, snapshot_date) DO UPDATE SET status = EXCLUDED.status, price_rub = EXCLUDED.price_rub, - price_per_m2 = EXCLUDED.price_per_m2, + -- COALESCE: keep a backfilled / sale_graph-derived price_per_m2 when the + -- scraper produces NULL (NULL-status historical flats have no price_rub). + -- Direct scraper value always wins when non-NULL. + price_per_m2 = COALESCE(EXCLUDED.price_per_m2, domrf_kn_flats.price_per_m2), obj_id = EXCLUDED.obj_id, region_cd = EXCLUDED.region_cd, obj_name = EXCLUDED.obj_name, diff --git a/data/sql/120_backfill_kn_price_per_m2.sql b/data/sql/120_backfill_kn_price_per_m2.sql new file mode 100644 index 00000000..ce57f0d5 --- /dev/null +++ b/data/sql/120_backfill_kn_price_per_m2.sql @@ -0,0 +1,76 @@ +-- 120_backfill_kn_price_per_m2.sql +-- Context: Issue #573 — domrf_kn_flats.price_per_m2 is populated in only ~3.7% +-- of rows (status='free'/'sold'/'booked'). The 756 977 NULL-status rows imported +-- from historical DOM.RF snapshots carry total_area but no price_rub, so the +-- scraper's price_rub/total_area derivation never fires for them. +-- This leaves 5/8 EKB districts with 0% price_per_m2 coverage and NULL district +-- medians, blocking the Limit_District_Median_Partial analytics limitation. +-- +-- Strategy (three-pass, most-precise source first): +-- Pass 1 — direct arithmetic: price_rub / total_area where both columns present +-- and price_per_m2 is still NULL or zero. Covers any rows the scraper +-- missed (idempotent sanity net). +-- Pass 2 — sale_graph join: use domrf_kn_sale_graph.price_avg (type='apartments', +-- latest snapshot) as a proxy for the object-level avg ₽/м². Applied +-- only to rows where Pass 1 left price_per_m2 NULL and the object has +-- a sale_graph row. +-- Pass 3 — peer median: for objects without sale_graph, derive price_per_m2 from +-- the PERCENTILE_CONT(0.5) of free/sold/booked flats in the same obj_id. +-- +-- Idempotent: WHERE price_per_m2 IS NULL guard prevents re-overwriting correct values. +-- Dependencies: domrf_kn_flats, domrf_kn_sale_graph (migration 50_schema_kn_extensions.sql). +-- Apply order: after 99_nspd_entities_denorm.sql. +-- Estimated runtime on prod: < 30 s (785 K rows, index on obj_id exists). + +BEGIN; + +-- ── Pass 1: direct price_rub / total_area ──────────────────────────────────── +UPDATE public.domrf_kn_flats + SET price_per_m2 = ROUND(price_rub / total_area, 2) + WHERE price_rub IS NOT NULL + AND total_area IS NOT NULL + AND total_area > 0 + AND (price_per_m2 IS NULL OR price_per_m2 = 0); + +-- ── Pass 2: sale_graph proxy (latest snapshot, type='apartments') ───────────── +-- domrf_kn_sale_graph.price_avg is the average contract ₽/м² reported by DOM.RF +-- for the object in a given month. We use the most recent snapshot's average +-- across all apartment sub-rows as a reasonable object-level proxy. +WITH sg_latest AS ( + SELECT obj_id, + AVG(price_avg) AS avg_ppm2 + FROM public.domrf_kn_sale_graph + WHERE type = 'apartments' + AND price_avg > 50000 + AND snapshot_date = ( + SELECT MAX(snapshot_date) + FROM public.domrf_kn_sale_graph + WHERE type = 'apartments' + ) + GROUP BY obj_id +) +UPDATE public.domrf_kn_flats f + SET price_per_m2 = ROUND(sg.avg_ppm2, 2) + FROM sg_latest sg + WHERE f.obj_id = sg.obj_id + AND f.price_per_m2 IS NULL; + +-- ── Pass 3: peer median from free/sold/booked flats in same obj ─────────────── +-- Fallback for objects absent from sale_graph: compute the object-level median +-- from the priced (status IS NOT NULL) flats in the same building. +WITH peer AS ( + SELECT obj_id, + PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY price_per_m2) AS median_ppm2 + FROM public.domrf_kn_flats + WHERE status IS NOT NULL + AND price_per_m2 IS NOT NULL + AND price_per_m2 > 0 + GROUP BY obj_id +) +UPDATE public.domrf_kn_flats f + SET price_per_m2 = ROUND(p.median_ppm2::numeric, 2) + FROM peer p + WHERE f.obj_id = p.obj_id + AND f.price_per_m2 IS NULL; + +COMMIT;