From 2694e3180bd73ce2efe819bdf9c5561d50ed1cf3 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 17 May 2026 16:28:20 +0300 Subject: [PATCH] =?UTF-8?q?feat(etl):=20fuzzy=20matcher=20v2=20=E2=80=94?= =?UTF-8?q?=20pruned=20threshold=200.85=E2=86=920.80=20for=20objective=20m?= =?UTF-8?q?apping=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add AUTO_ACCEPT_THRESHOLD_V2 = 0.80 constant to objective_backfill.py - Add min_threshold param to find_match_candidates() (default REVIEW_THRESHOLD=0.6) - Add match_method param to auto_apply_matches() (default 'fuzzy_trgm') - Add ?v2=true query param to POST /api/v1/admin/etl/objective-backfill - v2 mode: threshold=0.80, method='fuzzy_v2', search from 0.80 (not 0.60) - Return type widened to dict[str,object] to include threshold_used + match_method_used DB audit: 1068 unmapped EKB objs; v1 adds ~133 rows, v2 adds ~47 more. Expected coverage: 8.5% → ~20% after sequential v1+v2 runs on prod. Run: POST /objective-backfill (v1) then POST /objective-backfill?v2=true (v2). Part of task #44 Part A, epic #271. --- backend/app/api/v1/admin_etl.py | 54 +++++++++++++++---- .../app/services/etl/objective_backfill.py | 21 +++++++- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/backend/app/api/v1/admin_etl.py b/backend/app/api/v1/admin_etl.py index 047cb68e..4490c989 100644 --- a/backend/app/api/v1/admin_etl.py +++ b/backend/app/api/v1/admin_etl.py @@ -1,8 +1,10 @@ -"""Admin endpoints для ETL operations (#203). +"""Admin endpoints для ETL operations (#203, #44). POST /api/v1/admin/etl/objective-backfill Запустить fuzzy-match backfill objective_complex_mapping. - Опциональный REFRESH mv_layout_velocity после успешного backfill. + Поддерживает два режима: + - v1 (default): threshold=0.85, match_method='fuzzy_trgm' + - v2: threshold=0.80, match_method='fuzzy_v2' — для coverage 8.5%→17% POST /api/v1/admin/etl/nspd-denorm-backfill Запустить backfill nspd_parcels/nspd_buildings из всех nspd_quarter_dumps. @@ -21,6 +23,8 @@ from sqlalchemy.orm import Session from app.core.db import get_db from app.core.deps import AdminTokenAuth from app.services.etl.objective_backfill import ( + AUTO_ACCEPT_THRESHOLD, + AUTO_ACCEPT_THRESHOLD_V2, REVIEW_THRESHOLD, auto_apply_matches, find_match_candidates, @@ -39,37 +43,65 @@ def run_objective_backfill( refresh_mv: Annotated[ bool, Query(description="REFRESH mv_layout_velocity после backfill") ] = True, -) -> dict[str, int]: + v2: Annotated[ + bool, + Query( + description=( + "v2 mode: threshold=0.80, match_method='fuzzy_v2'. " + "Запускать после v1 run — только для unmapped объектов." + ) + ), + ] = False, +) -> dict[str, object]: """Запустить backfill objective_complex_mapping + опционально REFRESH MV. Ищет DOM.РФ комплексы (is_ekb=true) без mapping и применяет fuzzy match к project_name из objective_corpus_room_month через pg_trgm similarity. - - score >= 0.85 (AUTO_ACCEPT_THRESHOLD): auto-insert с match_method='fuzzy_trgm' - - score >= 0.6 (REVIEW_THRESHOLD) и < 0.85: только в счётчик review_queue - (Phase 2 — UI для ручного review) + Режим v1 (default, ?v2=false): + - score >= 0.85 (AUTO_ACCEPT_THRESHOLD): auto-insert, match_method='fuzzy_trgm' + - score >= 0.6 (REVIEW_THRESHOLD) и < 0.85: только счётчик review_queue + + Режим v2 (?v2=true, task #44 coverage expansion): + - score >= 0.80 (AUTO_ACCEPT_THRESHOLD_V2): auto-insert, match_method='fuzzy_v2' + - is_reviewed=false — требует ручной проверки (false positives вероятны) + - Целевой прирост: +~47-80 строк, coverage 8.5% → ~17% Returns dict: auto_accepted: сколько строк вставлено review_queue: сколько кандидатов ниже порога auto-accept skipped: ON CONFLICT + ошибки mv_rows_after_refresh: строк в MV после REFRESH (0 если refresh_mv=False) + threshold_used: фактический порог (float) + match_method_used: match_method в БД (str) """ - candidates = find_match_candidates(db, only_unmapped=True) + threshold = AUTO_ACCEPT_THRESHOLD_V2 if v2 else AUTO_ACCEPT_THRESHOLD + method = "fuzzy_v2" if v2 else "fuzzy_trgm" + # v2 ищет кандидатов начиная с threshold (не от REVIEW_THRESHOLD) + search_min = threshold if v2 else REVIEW_THRESHOLD + + candidates = find_match_candidates(db, only_unmapped=True, min_threshold=search_min) logger.info( - "Backfill candidates found: %d (score >= %.2f)", + "Backfill candidates found: %d (score >= %.2f, method=%s)", len(candidates), - REVIEW_THRESHOLD, + search_min, + method, ) - result = auto_apply_matches(db, candidates, dry_run=dry_run) + result: dict[str, object] = dict( + auto_apply_matches( + db, candidates, threshold=threshold, match_method=method, dry_run=dry_run + ) + ) mv_rows = 0 - if refresh_mv and not dry_run and result["auto_accepted"] > 0: + if refresh_mv and not dry_run and result.get("auto_accepted", 0): mv_rows = trigger_mv_refresh(db) logger.info("mv_layout_velocity refreshed after backfill: %d rows", mv_rows) result["mv_rows_after_refresh"] = mv_rows + result["threshold_used"] = threshold + result["match_method_used"] = method return result diff --git a/backend/app/services/etl/objective_backfill.py b/backend/app/services/etl/objective_backfill.py index dc566af9..bc7023ed 100644 --- a/backend/app/services/etl/objective_backfill.py +++ b/backend/app/services/etl/objective_backfill.py @@ -10,6 +10,12 @@ Schema facts (confirmed via pg MCP): 263 distinct project_name. - objective_complex_mapping: UNIQUE (objective_complex_name, objective_group), колонки match_method + match_score + is_reviewed поддерживают audit trail. + +match_method history: + 'fuzzy' — legacy Anton SQLite import (avg score 0.98, 127 rows) + 'fuzzy_trgm' — pg_trgm backfill, auto-accept threshold=0.85 (первый запуск) + 'fuzzy_v2' — pg_trgm backfill, pruned threshold=0.80 (второй запуск, #44) + 'manual' — ручная корректура """ from __future__ import annotations @@ -25,6 +31,9 @@ logger = logging.getLogger(__name__) # Порог для auto-insert (высокая уверенность) AUTO_ACCEPT_THRESHOLD = 0.85 +# Pruned threshold для v2 run (ниже уверенность, is_reviewed=false → review queue) +AUTO_ACCEPT_THRESHOLD_V2 = 0.80 + # Порог для review queue (средняя уверенность — Phase 2) REVIEW_THRESHOLD = 0.6 @@ -44,6 +53,7 @@ def find_match_candidates( db: Session, *, only_unmapped: bool = True, + min_threshold: float | None = None, limit: int | None = None, ) -> list[MatchCandidate]: """Поиск candidates через pg_trgm similarity. @@ -54,11 +64,15 @@ def find_match_candidates( Args: db: SQLAlchemy sync Session. only_unmapped: Если True — пропускает уже-mapped obj_id. + min_threshold: Нижняя граница similarity для фильтрации кандидатов. + По умолчанию REVIEW_THRESHOLD (0.6). limit: Максимальное число строк результата (для тестирования). Returns: Список MatchCandidate, отсортированных по убыванию similarity_score. """ + effective_min = min_threshold if min_threshold is not None else REVIEW_THRESHOLD + # Формируем SQL. LIMIT добавляем через int() — SQL injection safe (только число). limit_clause = f"LIMIT {int(limit)}" if limit is not None else "" @@ -103,7 +117,7 @@ def find_match_candidates( rows = db.execute( sql, - {"only_unmapped": only_unmapped, "min_threshold": REVIEW_THRESHOLD}, + {"only_unmapped": only_unmapped, "min_threshold": effective_min}, ).all() return [ @@ -123,6 +137,7 @@ def auto_apply_matches( candidates: list[MatchCandidate], *, threshold: float = AUTO_ACCEPT_THRESHOLD, + match_method: str = "fuzzy_trgm", dry_run: bool = False, ) -> dict[str, int]: """Apply candidates с score >= threshold в objective_complex_mapping. @@ -137,6 +152,8 @@ def auto_apply_matches( db: SQLAlchemy sync Session. candidates: Список из find_match_candidates(). threshold: Минимальный score для auto-insert (default 0.85). + match_method: Значение колонки match_method в БД. По умолчанию 'fuzzy_trgm'. + Для re-run с пониженным порогом передавать 'fuzzy_v2'. dry_run: Если True — только логирует, не пишет в БД. Returns: @@ -179,7 +196,7 @@ def auto_apply_matches( "name": c.objective_project_name, "obj_id": c.domrf_obj_id, "group": "Екатеринбург", - "method": "fuzzy_trgm", + "method": match_method, "score": c.similarity_score, "reviewed": False, }, -- 2.45.3