diff --git a/data/sql/160_refresh_ekb_districts_median_dedup.sql b/data/sql/160_refresh_ekb_districts_median_dedup.sql new file mode 100644 index 00000000..c8dc3d68 --- /dev/null +++ b/data/sql/160_refresh_ekb_districts_median_dedup.sql @@ -0,0 +1,91 @@ +-- Fix: dedup deals-to-district assignment in ekb_districts median refresh. +-- +-- Context: 67_refresh_ekb_districts_median.sql introduced ST_Intersects(district.geom, quarter.geom) +-- to join cadastral quarters to districts. A quarter whose polygon straddles a district boundary +-- matched BOTH districts, causing its deals to be counted in multiple districts and skewing +-- the PERCENTILE_CONT / AVG aggregates. +-- +-- Fix approach: replace ST_Intersects with ST_Within(ST_Centroid(cq.geom), d.geom). +-- The centroid of each cadastral quarter falls in exactly one district polygon, guaranteeing +-- a 1:1 quarter→district assignment. A DISTINCT ON fallback (ordered by d.district_name) is +-- added in case district polygons overlap at all (they should not, but defensive). +-- +-- Measured impact (prod, 2026-06-17, 24-month window): +-- ST_Intersects → 2945 deal-rows (2482 quarter-district pairs, 85 duplicate quarter pairs) +-- ST_Within(centroid) → 2638 deal-rows (2396 quarter-district pairs, 0 duplicates) +-- Eliminated: 307 spurious deal-district assignments. +-- +-- Dependencies: ekb_districts_geom (geom), cad_quarters_geom (geom, cad_number), +-- rosreestr_deals (quarter_cad_number, region_code, doc_type, +-- realestate_type_code, area, price_per_sqm, period_start_date), +-- ekb_districts (district_name, median_price_per_m2, mean_price_per_m2). +-- +-- 67_refresh_ekb_districts_median.sql ran on prod; this file supersedes it via _schema_migrations. +-- No dependent views or FK constraints on the function itself. +-- Signature unchanged — all existing callers (analytics_refresh.py, beat_schedule.py, etc.) +-- continue to work without modification. +-- +-- Deploy order: this file only (no backend code change required). +-- Idempotent: CREATE OR REPLACE FUNCTION — safe to re-run. + +BEGIN; + +CREATE OR REPLACE FUNCTION refresh_ekb_districts_median(window_months int DEFAULT 24, min_deals int DEFAULT 50) +RETURNS TABLE(district_name text, deals_used bigint, median_pm numeric, mean_pm numeric) +LANGUAGE plpgsql AS $$ +BEGIN + RETURN QUERY + WITH + -- Assign each cadastral quarter to exactly ONE district via centroid containment. + -- ST_Within(centroid, district_geom) is 1:1 whereas ST_Intersects matched border- + -- straddling quarters to multiple districts, inflating counts and skewing medians. + -- DISTINCT ON (cq.cad_number) with deterministic ORDER BY guards against the edge + -- case of overlapping district polygons. + quarter_district AS ( + SELECT DISTINCT ON (cq.cad_number) + cq.cad_number, + d.district_name + FROM cad_quarters_geom cq + JOIN ekb_districts_geom d + ON ST_Within(ST_Centroid(cq.geom), d.geom) + WHERE d.district_name <> 'не определён' + ORDER BY cq.cad_number, d.district_name + ), + per_district AS ( + SELECT qd.district_name, + COUNT(*)::bigint AS deals, + PERCENTILE_CONT(0.5) WITHIN GROUP (ORDER BY rd.price_per_sqm)::numeric AS p_median, + AVG(rd.price_per_sqm)::numeric AS p_mean + FROM rosreestr_deals rd + JOIN quarter_district qd ON qd.cad_number = rd.quarter_cad_number + WHERE rd.region_code = 66 + AND rd.doc_type = 'ДДУ' + AND rd.realestate_type_code = '002001003000' + AND rd.area BETWEEN 15 AND 200 + AND rd.price_per_sqm BETWEEN 30000 AND 1000000 + AND rd.period_start_date >= NOW() - (window_months || ' months')::interval + GROUP BY qd.district_name + ), + upd AS ( + UPDATE ekb_districts e + SET median_price_per_m2 = ROUND(pd.p_median, 0), + mean_price_per_m2 = ROUND(pd.p_mean, 0) + FROM per_district pd + WHERE e.district_name = pd.district_name + AND e.district_name <> 'не определён' + AND pd.deals >= min_deals + RETURNING e.district_name, pd.deals, pd.p_median, pd.p_mean + ) + SELECT u.district_name::text, u.deals, ROUND(u.p_median, 0), ROUND(u.p_mean, 0) + FROM upd u + ORDER BY u.district_name; +END; +$$; + +COMMENT ON FUNCTION refresh_ekb_districts_median(int, int) IS + 'Refresh ekb_districts.median_price_per_m2 + mean_price_per_m2 из ДДУ-сделок rosreestr_deals. ' + 'Spatial join: ST_Within(ST_Centroid(cad_quarter), district) — каждый квартал ровно в одном районе ' + '(fixes ST_Intersects double-count на граничных кварталах; issue #1352). ' + 'Окно по умолчанию 24 мес, минимум 50 сделок на район. Запускается Celery beat раз в месяц.'; + +COMMIT;