Merge pull request 'fix(site-finder): gate velocity on confidence of objective_complex_mapping (#307 OBJ-2)' (#1720) from fix/objective-mapping-is-reviewed-gate-307-obj2 into main

backend/app/services/site_finder/velocity.py

@@ -16,9 +16,13 @@ Foundation: domrf_kn_objects (lat/lon, comm_name, obj_class, region_cd),
 Fallback:   rosreestr_deals (quarter_cad_number, deal_count, period_start_date).
 
 Linkage: domrf_kn_objects.obj_id
-         → objective_complex_mapping.domrf_obj_id
+         → objective_complex_mapping.domrf_obj_id  (gated: is_reviewed/manual/score≥0.85)
          → objective_complex_mapping.objective_complex_name
          → objective_corpus_room_month.project_name
+
+OBJ-2 (#307): маппинги фильтруются по confidence (_MAPPING_CONFIDENCE_GATE) —
+unreviewed low-score auto-matches (#1331/#1333 backfill) исключаются как
+false-positive risk.
 """
 
 from __future__ import annotations
@@ -40,6 +44,24 @@ _EKB_MEDIAN_FALLBACK_SQM_PER_MONTH: float = 4500.0
 # пытаемся rosreestr_fallback.
 _OBJECTIVE_COVERAGE_MIN_RATIO: float = 0.50
 
+# OBJ-2 (#307): gate objective_complex_mapping by confidence перед использованием
+# в velocity. Fuzzy-trgm backfill (#1331/#1333) добавил ~115 auto-matched строк с
+# is_reviewed=false и низким match_score (вплоть до 0.50-0.625) — false-positive
+# risk, который раздувал/искажал velocity конкурентов.
+#
+# Принимаем mapping только если:
+#   - is_reviewed = TRUE          (человек подтвердил), ИЛИ
+#   - match_method = 'manual'     (ручной маппинг, score обычно NULL), ИЛИ
+#   - match_score >= 0.85         (AUTO_ACCEPT_THRESHOLD — high-confidence auto,
+#                                  vault: fuzzy_trgm 0.85+ надёжен для auto-use).
+#
+# Строгий gate только на is_reviewed=true дал бы 2 строки из 303 → обнулил бы
+# velocity-покрытие; 0.85-порог сохраняет 264/303 EKB-маппингов, отбрасывая 39
+# низкоуверенных. Совпадает с AUTO_ACCEPT_THRESHOLD из objective_backfill.py.
+_MAPPING_CONFIDENCE_GATE: str = (
+    "(cm.is_reviewed = TRUE OR cm.match_method = 'manual' OR cm.match_score >= 0.85)"
+)
+
 
 @dataclass(frozen=True)
 class VelocityResult:
@@ -195,7 +217,7 @@ def compute_velocity(
             sales_rows = (
                 db.execute(
                     text(
-                        """
+                        f"""
                     WITH all_competitors AS (
                         SELECT unnest(CAST(:obj_ids AS int[])) AS obj_id
                     ),
@@ -204,6 +226,7 @@ def compute_velocity(
                                cm.objective_complex_name
                           FROM objective_complex_mapping cm
                          WHERE cm.domrf_obj_id = ANY(:obj_ids)
+                           AND {_MAPPING_CONFIDENCE_GATE}
                     )
                     SELECT
                         ac.obj_id,
@@ -311,12 +334,13 @@ def compute_velocity(
             bucket_rows = (
                 db.execute(
                     text(
-                        """
+                        f"""
                     WITH mapped AS (
                         SELECT cm.domrf_obj_id AS obj_id,
                                cm.objective_complex_name
                           FROM objective_complex_mapping cm
                          WHERE cm.domrf_obj_id = ANY(:obj_ids)
+                           AND {_MAPPING_CONFIDENCE_GATE}
                     )
                     SELECT
                         m.obj_id,

backend/tests/test_velocity.py

@@ -327,6 +327,39 @@ def test_by_room_bucket_empty_when_no_bucket_data():
     assert result.by_room_bucket == {}
 
 
+def test_mapping_confidence_gate_in_sales_query():
+    """OBJ-2 (#307): sales/bucket queries фильтруют objective_complex_mapping
+    по confidence — unreviewed low-score auto-matches исключены.
+
+    Проверяем, что SQL, переданный в db.execute для маппинг-CTE, содержит
+    предикат gate (is_reviewed / manual / match_score >= 0.85). Это гарантирует,
+    что ~115 fuzzy-trgm строк с is_reviewed=false и score<0.85 не попадают в velocity.
+    """
+    comp_rows = [_comp_row(1), _comp_row(2)]
+    sales_rows = [
+        _sales_row(1, total_sqm=4000.0, months=4),
+        _sales_row(2, total_sqm=3000.0, months=4),
+    ]
+    bucket_rows = [_bucket_row(1, "1", units_sold=20, sqm_sold=900.0)]
+    db = _make_db(comp_rows=comp_rows, sales_rows=sales_rows, bucket_rows=bucket_rows)
+
+    with patch(
+        "app.services.site_finder.velocity._get_ekb_median",
+        return_value=_EKB_MEDIAN_FALLBACK_SQM_PER_MONTH,
+    ):
+        compute_velocity(db, parcel_geom_wkt=_PARCEL_WKT)
+
+    # db.execute вызывается 3 раза: comp / sales / bucket. SQL берём из call_args.
+    executed_sql = [str(call.args[0]) for call in db.execute.call_args_list]
+    # comp-query НЕ трогает mapping; sales (idx 1) и bucket (idx 2) — должны иметь gate.
+    mapping_queries = [s for s in executed_sql if "objective_complex_mapping" in s]
+    assert len(mapping_queries) >= 2, "ожидаются sales + bucket запросы с mapping"
+    for sql in mapping_queries:
+        assert "cm.is_reviewed = TRUE" in sql
+        assert "cm.match_method = 'manual'" in sql
+        assert "cm.match_score >= 0.85" in sql
+
+
 def test_sample_competitors_include_by_room_bucket():
     """sample_competitors каждого элемента содержит by_room_bucket."""
     comp_rows = [_comp_row(1), _comp_row(2)]

data/sql/165_velocity_mapping_reviewed_gate.sql

@@ -0,0 +1,93 @@
+-- 165_velocity_mapping_reviewed_gate.sql
+-- #307 OBJ-2 — gate objective_complex_mapping by confidence в mv_layout_velocity.
+--
+-- Контекст: fuzzy-trgm backfill (#1331/#1333, миграции 155/156/116/117/150) добавил
+-- ~115+ auto-matched строк в objective_complex_mapping с is_reviewed=false и низким
+-- match_score (вплоть до 0.50-0.625). Они с высокой вероятностью false-positive
+-- (Objective project name ≠ тот же ЖК что domrf_obj_id) и искажали velocity-агрегаты
+-- Site Finder, попадая в MV наравне с проверенными маппингами.
+--
+-- Fix: redefine MV с confidence-gate в JOIN — принимаем маппинг только если
+--   is_reviewed = TRUE          (человек подтвердил), ИЛИ
+--   match_method = 'manual'     (ручной маппинг, score обычно NULL), ИЛИ
+--   match_score >= 0.85         (AUTO_ACCEPT_THRESHOLD из objective_backfill.py —
+--                               high-confidence auto, fuzzy_trgm 0.85+ надёжен).
+--
+-- Строгий gate только на is_reviewed=true дал бы 2 строки из 303 EKB-маппингов →
+-- обнулил бы velocity. Порог 0.85 сохраняет ~264/303, отбрасывая ~39 низкоуверенных.
+-- Тот же предикат применён в runtime-пути backend/app/services/site_finder/velocity.py.
+--
+-- Сохраняет weighted-average формулу из #21 (100_fix_mv_layout_velocity_weighted_avg.sql) —
+-- меняется ТОЛЬКО WHERE/JOIN-условие на objective_complex_mapping.
+--
+-- Dependencies: mv_layout_velocity не имеет зависимых VIEW/MV (проверено 2026-05-17/100_fix).
+-- Idempotency: DROP IF EXISTS → безопасен при повторном запуске.
+-- Deploy order: SQL migration → app deploy. Auto-applied через _schema_migrations.
+-- ВАЖНО: DROP + CREATE + REFRESH (non-concurrent, блокировка ~30 с на ~19 738 source rows).
+
+BEGIN;
+
+DROP MATERIALIZED VIEW IF EXISTS mv_layout_velocity CASCADE;
+
+CREATE MATERIALIZED VIEW mv_layout_velocity AS
+WITH last24mo AS (
+    SELECT
+        ocm.project_name,
+        CASE
+            WHEN ocm.room_bucket = 'студия' THEN 'studio'
+            ELSE ocm.room_bucket
+        END                                                      AS room_bucket,
+        ocm.deals_total_count,
+        ocm.deals_total_avg_area_m2,
+        ocm.deals_total_avg_price_thousand_rub_per_m2,
+        ocm.deals_total_vol_m2,
+        ocm.report_month
+    FROM objective_corpus_room_month ocm
+    WHERE ocm.report_month >= (NOW() - INTERVAL '24 months')::date
+)
+SELECT
+    cm.domrf_obj_id                                              AS obj_id,
+    l.room_bucket,
+    SUM(l.deals_total_count)::int                               AS total_deals_24mo,
+
+    -- weighted average (#21) — нули из месяцев без сделок не тянут вниз.
+    (SUM(l.deals_total_avg_area_m2 * l.deals_total_count)
+        / NULLIF(SUM(l.deals_total_count), 0))::numeric(10, 2) AS avg_area_m2,
+
+    (SUM(l.deals_total_avg_price_thousand_rub_per_m2 * l.deals_total_count)
+        / NULLIF(SUM(l.deals_total_count), 0))::numeric(12, 2) AS avg_price_thousand_rub_per_m2,
+
+    SUM(l.deals_total_vol_m2)::numeric(12, 2)                  AS total_vol_m2,
+    MIN(l.report_month)                                         AS window_start,
+    MAX(l.report_month)                                         AS window_end,
+    COUNT(DISTINCT l.report_month)::int                         AS months_with_data
+FROM last24mo l
+JOIN objective_complex_mapping cm
+    ON cm.objective_complex_name = l.project_name
+WHERE l.room_bucket IS NOT NULL
+  AND cm.domrf_obj_id IS NOT NULL
+  AND cm.objective_group = 'Екатеринбург'
+  -- #307 OBJ-2: confidence-gate — исключаем unreviewed low-score auto-matches.
+  AND (cm.is_reviewed = TRUE OR cm.match_method = 'manual' OR cm.match_score >= 0.85)
+GROUP BY cm.domrf_obj_id, l.room_bucket
+WITH NO DATA;
+
+-- UNIQUE index: required for REFRESH CONCURRENTLY (periodic via layout_velocity_refresh.py).
+CREATE UNIQUE INDEX mv_layout_velocity_pk
+    ON mv_layout_velocity (obj_id, room_bucket);
+
+-- Lookup index for /best-layouts endpoint queries by obj_id.
+CREATE INDEX mv_layout_velocity_obj_idx
+    ON mv_layout_velocity (obj_id);
+
+-- Initial populate (non-concurrent — MV just created, CONCURRENTLY requires populated MV).
+REFRESH MATERIALIZED VIEW mv_layout_velocity;
+
+COMMENT ON MATERIALIZED VIEW mv_layout_velocity IS
+    'Per-(obj_id, room_bucket) deals aggregation за last 24 months. '
+    'WEIGHTED average площади и цены (SF Bug #21). '
+    'Confidence-gated mapping: is_reviewed/manual/score>=0.85 (#307 OBJ-2). '
+    'Source: objective_corpus_room_month × objective_complex_mapping (EKB only). '
+    'Refresh via layout_velocity_refresh.py (CONCURRENTLY после initial populate).';
+
+COMMIT;