gendesign/backend/app/services/etl/objective_backfill.py

"""Backfill objective_complex_mapping (#203).

Auto-match DOM.РФ комплексов к Objective dataset через fuzzy name matching.
Goal: coverage 3% → 40%+ для mv_layout_velocity.

Schema facts (confirmed via pg MCP):
- domrf_kn_objects: фильтр is_ekb = true (~1285 ЕКБ объектов).
  district_name ILIKE '%екатеринбург%' возвращает 0 строк — не использовать.
- objective_corpus_room_month: group_name = 'Екатеринбург' (единственное значение),
  263 distinct project_name.
- objective_complex_mapping: UNIQUE (objective_complex_name, objective_group),
  колонки match_method + match_score + is_reviewed поддерживают audit trail.

match_method history:
  'fuzzy'      — legacy Anton SQLite import (avg score 0.98, 127 rows)
  'fuzzy_trgm' — pg_trgm backfill, auto-accept threshold=0.85 (первый запуск)
  'fuzzy_v2'   — pg_trgm backfill, pruned threshold=0.80 (второй запуск, #44)
  'manual'     — ручная корректура
"""

from __future__ import annotations

import logging
from dataclasses import dataclass

from sqlalchemy import text
from sqlalchemy.orm import Session

logger = logging.getLogger(__name__)

# Порог для auto-insert (высокая уверенность)
AUTO_ACCEPT_THRESHOLD = 0.85

# Pruned threshold для v2 run (ниже уверенность, is_reviewed=false → review queue)
AUTO_ACCEPT_THRESHOLD_V2 = 0.80

# Порог для review queue (средняя уверенность — Phase 2)
REVIEW_THRESHOLD = 0.6


@dataclass
class MatchCandidate:
    """Один candidate match DOM.РФ ↔ Objective."""

    domrf_obj_id: int
    domrf_comm_name: str
    domrf_dev_name: str | None
    objective_project_name: str
    similarity_score: float  # 0.0..1.0


def find_match_candidates(
    db: Session,
    *,
    only_unmapped: bool = True,
    min_threshold: float | None = None,
    limit: int | None = None,
) -> list[MatchCandidate]:
    """Поиск candidates через pg_trgm similarity.

    Использует CROSS JOIN LATERAL + similarity() для fuzzy match
    comm_name (DOM.РФ) ↔ project_name (Objective).

    Args:
        db: SQLAlchemy sync Session.
        only_unmapped: Если True — пропускает уже-mapped obj_id.
        min_threshold: Нижняя граница similarity для фильтрации кандидатов.
            По умолчанию REVIEW_THRESHOLD (0.6).
        limit: Максимальное число строк результата (для тестирования).

    Returns:
        Список MatchCandidate, отсортированных по убыванию similarity_score.
    """
    effective_min = min_threshold if min_threshold is not None else REVIEW_THRESHOLD

    # Формируем SQL. LIMIT добавляем через int() — SQL injection safe (только число).
    limit_clause = f"LIMIT {int(limit)}" if limit is not None else ""

    sql = text(
        f"""
        WITH domrf_unmapped AS (
            SELECT o.obj_id, o.comm_name, o.dev_name
            FROM domrf_kn_objects o
            WHERE o.is_ekb = true
              AND o.comm_name IS NOT NULL
              AND (
                  CAST(:only_unmapped AS boolean) = FALSE
                  OR NOT EXISTS (
                      SELECT 1 FROM objective_complex_mapping cm
                      WHERE cm.domrf_obj_id = o.obj_id
                  )
              )
        ),
        objective_distinct AS (
            SELECT DISTINCT project_name
            FROM objective_corpus_room_month
            WHERE group_name = 'Екатеринбург'
        )
        SELECT
            d.obj_id,
            d.comm_name,
            d.dev_name,
            obj.project_name,
            similarity(d.comm_name, obj.project_name) AS sim_score
        FROM domrf_unmapped d
        CROSS JOIN LATERAL (
            SELECT project_name
            FROM objective_distinct
            WHERE similarity(d.comm_name, project_name) > 0.4
            ORDER BY similarity(d.comm_name, project_name) DESC
            LIMIT 1
        ) obj
        WHERE similarity(d.comm_name, obj.project_name) >= CAST(:min_threshold AS float)
        ORDER BY sim_score DESC
        {limit_clause}
        """
    )

    rows = db.execute(
        sql,
        {"only_unmapped": only_unmapped, "min_threshold": effective_min},
    ).all()

    return [
        MatchCandidate(
            domrf_obj_id=int(r[0]),
            domrf_comm_name=str(r[1]),
            domrf_dev_name=str(r[2]) if r[2] is not None else None,
            objective_project_name=str(r[3]),
            similarity_score=float(r[4]),
        )
        for r in rows
    ]


def auto_apply_matches(
    db: Session,
    candidates: list[MatchCandidate],
    *,
    threshold: float = AUTO_ACCEPT_THRESHOLD,
    match_method: str = "fuzzy_trgm",
    dry_run: bool = False,
) -> dict[str, int]:
    """Apply candidates с score >= threshold в objective_complex_mapping.

    Кандидаты ниже threshold, но >= REVIEW_THRESHOLD попадают в review_queue
    (Phase 2 — UI для ручного review, в этом PR только считаются).

    ON CONFLICT DO NOTHING — если пара (objective_complex_name, objective_group)
    уже существует, строка пропускается без ошибки.

    Args:
        db: SQLAlchemy sync Session.
        candidates: Список из find_match_candidates().
        threshold: Минимальный score для auto-insert (default 0.85).
        match_method: Значение колонки match_method в БД. По умолчанию 'fuzzy_trgm'.
            Для re-run с пониженным порогом передавать 'fuzzy_v2'.
        dry_run: Если True — только логирует, не пишет в БД.

    Returns:
        dict с ключами auto_accepted, review_queue, skipped.
    """
    auto = [c for c in candidates if c.similarity_score >= threshold]
    review = [c for c in candidates if REVIEW_THRESHOLD <= c.similarity_score < threshold]

    if dry_run:
        logger.info(
            "DRY RUN: would auto-accept %d, review queue %d",
            len(auto),
            len(review),
        )
        return {"auto_accepted": len(auto), "review_queue": len(review), "skipped": 0}

    inserted = 0
    skipped = 0
    for c in auto:
        try:
            with db.begin_nested():
                result = db.execute(
                    text(
                        """
                        INSERT INTO objective_complex_mapping
                            (objective_complex_name, domrf_obj_id, objective_group,
                             match_method, match_score, is_reviewed)
                        VALUES (
                            CAST(:name AS text),
                            CAST(:obj_id AS bigint),
                            CAST(:group AS text),
                            CAST(:method AS text),
                            CAST(:score AS numeric),
                            CAST(:reviewed AS boolean)
                        )
                        ON CONFLICT (objective_complex_name, objective_group) DO NOTHING
                        """
                    ),
                    {
                        "name": c.objective_project_name,
                        "obj_id": c.domrf_obj_id,
                        "group": "Екатеринбург",
                        "method": match_method,
                        "score": c.similarity_score,
                        "reviewed": False,
                    },
                )
            if result.rowcount > 0:
                inserted += 1
            else:
                skipped += 1
        except Exception as e:
            logger.warning(
                "Insert failed для %s ↔ %s: %s",
                c.domrf_comm_name,
                c.objective_project_name,
                e,
            )
            skipped += 1

    db.commit()
    logger.info(
        "Backfill complete: auto_accepted=%d skipped=%d review_queue=%d",
        inserted,
        skipped,
        len(review),
    )
    return {"auto_accepted": inserted, "review_queue": len(review), "skipped": skipped}


def trigger_mv_refresh(db: Session) -> int:
    """REFRESH mv_layout_velocity после backfill.

    Вызывает существующий helper из layout_velocity_refresh.
    Передаём concurrently=True (MV уже заполнен).
    """
    from app.services.site_finder.layout_velocity_refresh import refresh_layout_velocity

    return refresh_layout_velocity(db, concurrently=True)