diff --git a/backend/app/services/site_finder/velocity.py b/backend/app/services/site_finder/velocity.py index e391dee3..75b68784 100644 --- a/backend/app/services/site_finder/velocity.py +++ b/backend/app/services/site_finder/velocity.py @@ -16,9 +16,13 @@ Foundation: domrf_kn_objects (lat/lon, comm_name, obj_class, region_cd), Fallback: rosreestr_deals (quarter_cad_number, deal_count, period_start_date). Linkage: domrf_kn_objects.obj_id - → objective_complex_mapping.domrf_obj_id + → objective_complex_mapping.domrf_obj_id (gated: is_reviewed/manual/score≥0.85) → objective_complex_mapping.objective_complex_name → objective_corpus_room_month.project_name + +OBJ-2 (#307): маппинги фильтруются по confidence (_MAPPING_CONFIDENCE_GATE) — +unreviewed low-score auto-matches (#1331/#1333 backfill) исключаются как +false-positive risk. """ from __future__ import annotations @@ -40,6 +44,24 @@ _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH: float = 4500.0 # пытаемся rosreestr_fallback. _OBJECTIVE_COVERAGE_MIN_RATIO: float = 0.50 +# OBJ-2 (#307): gate objective_complex_mapping by confidence перед использованием +# в velocity. Fuzzy-trgm backfill (#1331/#1333) добавил ~115 auto-matched строк с +# is_reviewed=false и низким match_score (вплоть до 0.50-0.625) — false-positive +# risk, который раздувал/искажал velocity конкурентов. +# +# Принимаем mapping только если: +# - is_reviewed = TRUE (человек подтвердил), ИЛИ +# - match_method = 'manual' (ручной маппинг, score обычно NULL), ИЛИ +# - match_score >= 0.85 (AUTO_ACCEPT_THRESHOLD — high-confidence auto, +# vault: fuzzy_trgm 0.85+ надёжен для auto-use). +# +# Строгий gate только на is_reviewed=true дал бы 2 строки из 303 → обнулил бы +# velocity-покрытие; 0.85-порог сохраняет 264/303 EKB-маппингов, отбрасывая 39 +# низкоуверенных. Совпадает с AUTO_ACCEPT_THRESHOLD из objective_backfill.py. +_MAPPING_CONFIDENCE_GATE: str = ( + "(cm.is_reviewed = TRUE OR cm.match_method = 'manual' OR cm.match_score >= 0.85)" +) + @dataclass(frozen=True) class VelocityResult: @@ -195,7 +217,7 @@ def compute_velocity( sales_rows = ( db.execute( text( - """ + f""" WITH all_competitors AS ( SELECT unnest(CAST(:obj_ids AS int[])) AS obj_id ), @@ -204,6 +226,7 @@ def compute_velocity( cm.objective_complex_name FROM objective_complex_mapping cm WHERE cm.domrf_obj_id = ANY(:obj_ids) + AND {_MAPPING_CONFIDENCE_GATE} ) SELECT ac.obj_id, @@ -311,12 +334,13 @@ def compute_velocity( bucket_rows = ( db.execute( text( - """ + f""" WITH mapped AS ( SELECT cm.domrf_obj_id AS obj_id, cm.objective_complex_name FROM objective_complex_mapping cm WHERE cm.domrf_obj_id = ANY(:obj_ids) + AND {_MAPPING_CONFIDENCE_GATE} ) SELECT m.obj_id, diff --git a/backend/tests/test_velocity.py b/backend/tests/test_velocity.py index aa25fdbb..c5aff67b 100644 --- a/backend/tests/test_velocity.py +++ b/backend/tests/test_velocity.py @@ -327,6 +327,39 @@ def test_by_room_bucket_empty_when_no_bucket_data(): assert result.by_room_bucket == {} +def test_mapping_confidence_gate_in_sales_query(): + """OBJ-2 (#307): sales/bucket queries фильтруют objective_complex_mapping + по confidence — unreviewed low-score auto-matches исключены. + + Проверяем, что SQL, переданный в db.execute для маппинг-CTE, содержит + предикат gate (is_reviewed / manual / match_score >= 0.85). Это гарантирует, + что ~115 fuzzy-trgm строк с is_reviewed=false и score<0.85 не попадают в velocity. + """ + comp_rows = [_comp_row(1), _comp_row(2)] + sales_rows = [ + _sales_row(1, total_sqm=4000.0, months=4), + _sales_row(2, total_sqm=3000.0, months=4), + ] + bucket_rows = [_bucket_row(1, "1", units_sold=20, sqm_sold=900.0)] + db = _make_db(comp_rows=comp_rows, sales_rows=sales_rows, bucket_rows=bucket_rows) + + with patch( + "app.services.site_finder.velocity._get_ekb_median", + return_value=_EKB_MEDIAN_FALLBACK_SQM_PER_MONTH, + ): + compute_velocity(db, parcel_geom_wkt=_PARCEL_WKT) + + # db.execute вызывается 3 раза: comp / sales / bucket. SQL берём из call_args. + executed_sql = [str(call.args[0]) for call in db.execute.call_args_list] + # comp-query НЕ трогает mapping; sales (idx 1) и bucket (idx 2) — должны иметь gate. + mapping_queries = [s for s in executed_sql if "objective_complex_mapping" in s] + assert len(mapping_queries) >= 2, "ожидаются sales + bucket запросы с mapping" + for sql in mapping_queries: + assert "cm.is_reviewed = TRUE" in sql + assert "cm.match_method = 'manual'" in sql + assert "cm.match_score >= 0.85" in sql + + def test_sample_competitors_include_by_room_bucket(): """sample_competitors каждого элемента содержит by_room_bucket.""" comp_rows = [_comp_row(1), _comp_row(2)] diff --git a/data/sql/165_velocity_mapping_reviewed_gate.sql b/data/sql/165_velocity_mapping_reviewed_gate.sql new file mode 100644 index 00000000..1e2bb619 --- /dev/null +++ b/data/sql/165_velocity_mapping_reviewed_gate.sql @@ -0,0 +1,93 @@ +-- 165_velocity_mapping_reviewed_gate.sql +-- #307 OBJ-2 — gate objective_complex_mapping by confidence в mv_layout_velocity. +-- +-- Контекст: fuzzy-trgm backfill (#1331/#1333, миграции 155/156/116/117/150) добавил +-- ~115+ auto-matched строк в objective_complex_mapping с is_reviewed=false и низким +-- match_score (вплоть до 0.50-0.625). Они с высокой вероятностью false-positive +-- (Objective project name ≠ тот же ЖК что domrf_obj_id) и искажали velocity-агрегаты +-- Site Finder, попадая в MV наравне с проверенными маппингами. +-- +-- Fix: redefine MV с confidence-gate в JOIN — принимаем маппинг только если +-- is_reviewed = TRUE (человек подтвердил), ИЛИ +-- match_method = 'manual' (ручной маппинг, score обычно NULL), ИЛИ +-- match_score >= 0.85 (AUTO_ACCEPT_THRESHOLD из objective_backfill.py — +-- high-confidence auto, fuzzy_trgm 0.85+ надёжен). +-- +-- Строгий gate только на is_reviewed=true дал бы 2 строки из 303 EKB-маппингов → +-- обнулил бы velocity. Порог 0.85 сохраняет ~264/303, отбрасывая ~39 низкоуверенных. +-- Тот же предикат применён в runtime-пути backend/app/services/site_finder/velocity.py. +-- +-- Сохраняет weighted-average формулу из #21 (100_fix_mv_layout_velocity_weighted_avg.sql) — +-- меняется ТОЛЬКО WHERE/JOIN-условие на objective_complex_mapping. +-- +-- Dependencies: mv_layout_velocity не имеет зависимых VIEW/MV (проверено 2026-05-17/100_fix). +-- Idempotency: DROP IF EXISTS → безопасен при повторном запуске. +-- Deploy order: SQL migration → app deploy. Auto-applied через _schema_migrations. +-- ВАЖНО: DROP + CREATE + REFRESH (non-concurrent, блокировка ~30 с на ~19 738 source rows). + +BEGIN; + +DROP MATERIALIZED VIEW IF EXISTS mv_layout_velocity CASCADE; + +CREATE MATERIALIZED VIEW mv_layout_velocity AS +WITH last24mo AS ( + SELECT + ocm.project_name, + CASE + WHEN ocm.room_bucket = 'студия' THEN 'studio' + ELSE ocm.room_bucket + END AS room_bucket, + ocm.deals_total_count, + ocm.deals_total_avg_area_m2, + ocm.deals_total_avg_price_thousand_rub_per_m2, + ocm.deals_total_vol_m2, + ocm.report_month + FROM objective_corpus_room_month ocm + WHERE ocm.report_month >= (NOW() - INTERVAL '24 months')::date +) +SELECT + cm.domrf_obj_id AS obj_id, + l.room_bucket, + SUM(l.deals_total_count)::int AS total_deals_24mo, + + -- weighted average (#21) — нули из месяцев без сделок не тянут вниз. + (SUM(l.deals_total_avg_area_m2 * l.deals_total_count) + / NULLIF(SUM(l.deals_total_count), 0))::numeric(10, 2) AS avg_area_m2, + + (SUM(l.deals_total_avg_price_thousand_rub_per_m2 * l.deals_total_count) + / NULLIF(SUM(l.deals_total_count), 0))::numeric(12, 2) AS avg_price_thousand_rub_per_m2, + + SUM(l.deals_total_vol_m2)::numeric(12, 2) AS total_vol_m2, + MIN(l.report_month) AS window_start, + MAX(l.report_month) AS window_end, + COUNT(DISTINCT l.report_month)::int AS months_with_data +FROM last24mo l +JOIN objective_complex_mapping cm + ON cm.objective_complex_name = l.project_name +WHERE l.room_bucket IS NOT NULL + AND cm.domrf_obj_id IS NOT NULL + AND cm.objective_group = 'Екатеринбург' + -- #307 OBJ-2: confidence-gate — исключаем unreviewed low-score auto-matches. + AND (cm.is_reviewed = TRUE OR cm.match_method = 'manual' OR cm.match_score >= 0.85) +GROUP BY cm.domrf_obj_id, l.room_bucket +WITH NO DATA; + +-- UNIQUE index: required for REFRESH CONCURRENTLY (periodic via layout_velocity_refresh.py). +CREATE UNIQUE INDEX mv_layout_velocity_pk + ON mv_layout_velocity (obj_id, room_bucket); + +-- Lookup index for /best-layouts endpoint queries by obj_id. +CREATE INDEX mv_layout_velocity_obj_idx + ON mv_layout_velocity (obj_id); + +-- Initial populate (non-concurrent — MV just created, CONCURRENTLY requires populated MV). +REFRESH MATERIALIZED VIEW mv_layout_velocity; + +COMMENT ON MATERIALIZED VIEW mv_layout_velocity IS + 'Per-(obj_id, room_bucket) deals aggregation за last 24 months. ' + 'WEIGHTED average площади и цены (SF Bug #21). ' + 'Confidence-gated mapping: is_reviewed/manual/score>=0.85 (#307 OBJ-2). ' + 'Source: objective_corpus_room_month × objective_complex_mapping (EKB only). ' + 'Refresh via layout_velocity_refresh.py (CONCURRENTLY после initial populate).'; + +COMMIT;