"""Backfill objective_complex_mapping (#203). Auto-match DOM.РФ комплексов к Objective dataset через fuzzy name matching. Goal: coverage 3% → 40%+ для mv_layout_velocity. Schema facts (confirmed via pg MCP): - domrf_kn_objects: фильтр is_ekb = true (~1285 ЕКБ объектов). district_name ILIKE '%екатеринбург%' возвращает 0 строк — не использовать. - objective_corpus_room_month: group_name = 'Екатеринбург' (единственное значение), 263 distinct project_name. - objective_complex_mapping: UNIQUE (objective_complex_name, objective_group), колонки match_method + match_score + is_reviewed поддерживают audit trail. match_method history: 'fuzzy' — legacy Anton SQLite import (avg score 0.98, 127 rows) 'fuzzy_trgm' — pg_trgm backfill, auto-accept threshold=0.85 (первый запуск) 'fuzzy_v2' — pg_trgm backfill, pruned threshold=0.80 (второй запуск, #44) 'manual' — ручная корректура """ from __future__ import annotations import logging from dataclasses import dataclass from sqlalchemy import text from sqlalchemy.orm import Session logger = logging.getLogger(__name__) # Порог для auto-insert (высокая уверенность) AUTO_ACCEPT_THRESHOLD = 0.85 # Pruned threshold для v2 run (ниже уверенность, is_reviewed=false → review queue) AUTO_ACCEPT_THRESHOLD_V2 = 0.80 # Порог для review queue (средняя уверенность — Phase 2) REVIEW_THRESHOLD = 0.6 @dataclass class MatchCandidate: """Один candidate match DOM.РФ ↔ Objective.""" domrf_obj_id: int domrf_comm_name: str domrf_dev_name: str | None objective_project_name: str similarity_score: float # 0.0..1.0 def find_match_candidates( db: Session, *, only_unmapped: bool = True, min_threshold: float | None = None, limit: int | None = None, ) -> list[MatchCandidate]: """Поиск candidates через pg_trgm similarity. Использует CROSS JOIN LATERAL + similarity() для fuzzy match comm_name (DOM.РФ) ↔ project_name (Objective). Args: db: SQLAlchemy sync Session. only_unmapped: Если True — пропускает уже-mapped obj_id. min_threshold: Нижняя граница similarity для фильтрации кандидатов. По умолчанию REVIEW_THRESHOLD (0.6). limit: Максимальное число строк результата (для тестирования). Returns: Список MatchCandidate, отсортированных по убыванию similarity_score. """ effective_min = min_threshold if min_threshold is not None else REVIEW_THRESHOLD # Формируем SQL. LIMIT добавляем через int() — SQL injection safe (только число). limit_clause = f"LIMIT {int(limit)}" if limit is not None else "" sql = text( f""" WITH domrf_unmapped AS ( SELECT o.obj_id, o.comm_name, o.dev_name FROM domrf_kn_objects o WHERE o.is_ekb = true AND o.comm_name IS NOT NULL AND ( CAST(:only_unmapped AS boolean) = FALSE OR NOT EXISTS ( SELECT 1 FROM objective_complex_mapping cm WHERE cm.domrf_obj_id = o.obj_id ) ) ), objective_distinct AS ( SELECT DISTINCT project_name FROM objective_corpus_room_month ) SELECT d.obj_id, d.comm_name, d.dev_name, obj.project_name, similarity(d.comm_name, obj.project_name) AS sim_score FROM domrf_unmapped d CROSS JOIN LATERAL ( SELECT project_name FROM objective_distinct WHERE similarity(d.comm_name, project_name) > 0.4 ORDER BY similarity(d.comm_name, project_name) DESC LIMIT 1 ) obj WHERE similarity(d.comm_name, obj.project_name) >= CAST(:min_threshold AS float) ORDER BY sim_score DESC {limit_clause} """ ) rows = db.execute( sql, {"only_unmapped": only_unmapped, "min_threshold": effective_min}, ).all() return [ MatchCandidate( domrf_obj_id=int(r[0]), domrf_comm_name=str(r[1]), domrf_dev_name=str(r[2]) if r[2] is not None else None, objective_project_name=str(r[3]), similarity_score=float(r[4]), ) for r in rows ] def auto_apply_matches( db: Session, candidates: list[MatchCandidate], *, threshold: float = AUTO_ACCEPT_THRESHOLD, match_method: str = "fuzzy_trgm", dry_run: bool = False, ) -> dict[str, int]: """Apply candidates с score >= threshold в objective_complex_mapping. Кандидаты ниже threshold, но >= REVIEW_THRESHOLD попадают в review_queue (Phase 2 — UI для ручного review, в этом PR только считаются). ON CONFLICT DO NOTHING — если пара (objective_complex_name, objective_group) уже существует, строка пропускается без ошибки. Args: db: SQLAlchemy sync Session. candidates: Список из find_match_candidates(). threshold: Минимальный score для auto-insert (default 0.85). match_method: Значение колонки match_method в БД. По умолчанию 'fuzzy_trgm'. Для re-run с пониженным порогом передавать 'fuzzy_v2'. dry_run: Если True — только логирует, не пишет в БД. Returns: dict с ключами auto_accepted, review_queue, skipped. """ auto = [c for c in candidates if c.similarity_score >= threshold] review = [c for c in candidates if REVIEW_THRESHOLD <= c.similarity_score < threshold] if dry_run: logger.info( "DRY RUN: would auto-accept %d, review queue %d", len(auto), len(review), ) return {"auto_accepted": 0, "review_queue": len(review), "skipped": 0} inserted = 0 skipped = 0 for c in auto: try: with db.begin_nested(): result = db.execute( text( """ INSERT INTO objective_complex_mapping (objective_complex_name, domrf_obj_id, objective_group, match_method, match_score, is_reviewed) VALUES ( CAST(:name AS text), CAST(:obj_id AS bigint), CAST(:group AS text), CAST(:method AS text), CAST(:score AS numeric), CAST(:reviewed AS boolean) ) ON CONFLICT (objective_complex_name, objective_group) DO NOTHING """ ), { "name": c.objective_project_name, "obj_id": c.domrf_obj_id, "group": "Екатеринбург", "method": match_method, "score": c.similarity_score, "reviewed": False, }, ) if result.rowcount > 0: inserted += 1 else: skipped += 1 except Exception as e: logger.warning( "Insert failed для %s ↔ %s: %s", c.domrf_comm_name, c.objective_project_name, e, ) skipped += 1 db.commit() logger.info( "Backfill complete: auto_accepted=%d skipped=%d review_queue=%d", inserted, skipped, len(review), ) return {"auto_accepted": inserted, "review_queue": len(review), "skipped": skipped} def trigger_mv_refresh(db: Session) -> int: """REFRESH mv_layout_velocity после backfill. Вызывает существующий helper из layout_velocity_refresh. Передаём concurrently=True (MV уже заполнен). """ from app.services.site_finder.layout_velocity_refresh import refresh_layout_velocity return refresh_layout_velocity(db, concurrently=True)