bot-backend
14f3ef2019
Co-authored-by: bot-backend <bot-backend@gendsgn.local> Co-committed-by: bot-backend <bot-backend@gendsgn.local>
236 lines
8.6 KiB
Python
236 lines
8.6 KiB
Python
"""Backfill objective_complex_mapping (#203).
|
||
|
||
Auto-match DOM.РФ комплексов к Objective dataset через fuzzy name matching.
|
||
Goal: coverage 3% → 40%+ для mv_layout_velocity.
|
||
|
||
Schema facts (confirmed via pg MCP):
|
||
- domrf_kn_objects: фильтр is_ekb = true (~1285 ЕКБ объектов).
|
||
district_name ILIKE '%екатеринбург%' возвращает 0 строк — не использовать.
|
||
- objective_corpus_room_month: group_name = 'Екатеринбург' (единственное значение),
|
||
263 distinct project_name.
|
||
- objective_complex_mapping: UNIQUE (objective_complex_name, objective_group),
|
||
колонки match_method + match_score + is_reviewed поддерживают audit trail.
|
||
|
||
match_method history:
|
||
'fuzzy' — legacy Anton SQLite import (avg score 0.98, 127 rows)
|
||
'fuzzy_trgm' — pg_trgm backfill, auto-accept threshold=0.85 (первый запуск)
|
||
'fuzzy_v2' — pg_trgm backfill, pruned threshold=0.80 (второй запуск, #44)
|
||
'manual' — ручная корректура
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from dataclasses import dataclass
|
||
|
||
from sqlalchemy import text
|
||
from sqlalchemy.orm import Session
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# Порог для auto-insert (высокая уверенность)
|
||
AUTO_ACCEPT_THRESHOLD = 0.85
|
||
|
||
# Pruned threshold для v2 run (ниже уверенность, is_reviewed=false → review queue)
|
||
AUTO_ACCEPT_THRESHOLD_V2 = 0.80
|
||
|
||
# Порог для review queue (средняя уверенность — Phase 2)
|
||
REVIEW_THRESHOLD = 0.6
|
||
|
||
|
||
@dataclass
|
||
class MatchCandidate:
|
||
"""Один candidate match DOM.РФ ↔ Objective."""
|
||
|
||
domrf_obj_id: int
|
||
domrf_comm_name: str
|
||
domrf_dev_name: str | None
|
||
objective_project_name: str
|
||
similarity_score: float # 0.0..1.0
|
||
|
||
|
||
def find_match_candidates(
|
||
db: Session,
|
||
*,
|
||
only_unmapped: bool = True,
|
||
min_threshold: float | None = None,
|
||
limit: int | None = None,
|
||
) -> list[MatchCandidate]:
|
||
"""Поиск candidates через pg_trgm similarity.
|
||
|
||
Использует CROSS JOIN LATERAL + similarity() для fuzzy match
|
||
comm_name (DOM.РФ) ↔ project_name (Objective).
|
||
|
||
Args:
|
||
db: SQLAlchemy sync Session.
|
||
only_unmapped: Если True — пропускает уже-mapped obj_id.
|
||
min_threshold: Нижняя граница similarity для фильтрации кандидатов.
|
||
По умолчанию REVIEW_THRESHOLD (0.6).
|
||
limit: Максимальное число строк результата (для тестирования).
|
||
|
||
Returns:
|
||
Список MatchCandidate, отсортированных по убыванию similarity_score.
|
||
"""
|
||
effective_min = min_threshold if min_threshold is not None else REVIEW_THRESHOLD
|
||
|
||
# Формируем SQL. LIMIT добавляем через int() — SQL injection safe (только число).
|
||
limit_clause = f"LIMIT {int(limit)}" if limit is not None else ""
|
||
|
||
sql = text(
|
||
f"""
|
||
WITH domrf_unmapped AS (
|
||
SELECT o.obj_id, o.comm_name, o.dev_name
|
||
FROM domrf_kn_objects o
|
||
WHERE o.is_ekb = true
|
||
AND o.comm_name IS NOT NULL
|
||
AND (
|
||
CAST(:only_unmapped AS boolean) = FALSE
|
||
OR NOT EXISTS (
|
||
SELECT 1 FROM objective_complex_mapping cm
|
||
WHERE cm.domrf_obj_id = o.obj_id
|
||
)
|
||
)
|
||
),
|
||
objective_distinct AS (
|
||
SELECT DISTINCT project_name
|
||
FROM objective_corpus_room_month
|
||
WHERE group_name = 'Екатеринбург'
|
||
)
|
||
SELECT
|
||
d.obj_id,
|
||
d.comm_name,
|
||
d.dev_name,
|
||
obj.project_name,
|
||
similarity(d.comm_name, obj.project_name) AS sim_score
|
||
FROM domrf_unmapped d
|
||
CROSS JOIN LATERAL (
|
||
SELECT project_name
|
||
FROM objective_distinct
|
||
WHERE similarity(d.comm_name, project_name) > 0.4
|
||
ORDER BY similarity(d.comm_name, project_name) DESC
|
||
LIMIT 1
|
||
) obj
|
||
WHERE similarity(d.comm_name, obj.project_name) >= CAST(:min_threshold AS float)
|
||
ORDER BY sim_score DESC
|
||
{limit_clause}
|
||
"""
|
||
)
|
||
|
||
rows = db.execute(
|
||
sql,
|
||
{"only_unmapped": only_unmapped, "min_threshold": effective_min},
|
||
).all()
|
||
|
||
return [
|
||
MatchCandidate(
|
||
domrf_obj_id=int(r[0]),
|
||
domrf_comm_name=str(r[1]),
|
||
domrf_dev_name=str(r[2]) if r[2] is not None else None,
|
||
objective_project_name=str(r[3]),
|
||
similarity_score=float(r[4]),
|
||
)
|
||
for r in rows
|
||
]
|
||
|
||
|
||
def auto_apply_matches(
|
||
db: Session,
|
||
candidates: list[MatchCandidate],
|
||
*,
|
||
threshold: float = AUTO_ACCEPT_THRESHOLD,
|
||
match_method: str = "fuzzy_trgm",
|
||
dry_run: bool = False,
|
||
) -> dict[str, int]:
|
||
"""Apply candidates с score >= threshold в objective_complex_mapping.
|
||
|
||
Кандидаты ниже threshold, но >= REVIEW_THRESHOLD попадают в review_queue
|
||
(Phase 2 — UI для ручного review, в этом PR только считаются).
|
||
|
||
ON CONFLICT DO NOTHING — если пара (objective_complex_name, objective_group)
|
||
уже существует, строка пропускается без ошибки.
|
||
|
||
Args:
|
||
db: SQLAlchemy sync Session.
|
||
candidates: Список из find_match_candidates().
|
||
threshold: Минимальный score для auto-insert (default 0.85).
|
||
match_method: Значение колонки match_method в БД. По умолчанию 'fuzzy_trgm'.
|
||
Для re-run с пониженным порогом передавать 'fuzzy_v2'.
|
||
dry_run: Если True — только логирует, не пишет в БД.
|
||
|
||
Returns:
|
||
dict с ключами auto_accepted, review_queue, skipped.
|
||
"""
|
||
auto = [c for c in candidates if c.similarity_score >= threshold]
|
||
review = [c for c in candidates if REVIEW_THRESHOLD <= c.similarity_score < threshold]
|
||
|
||
if dry_run:
|
||
logger.info(
|
||
"DRY RUN: would auto-accept %d, review queue %d",
|
||
len(auto),
|
||
len(review),
|
||
)
|
||
return {"auto_accepted": len(auto), "review_queue": len(review), "skipped": 0}
|
||
|
||
inserted = 0
|
||
skipped = 0
|
||
for c in auto:
|
||
try:
|
||
with db.begin_nested():
|
||
result = db.execute(
|
||
text(
|
||
"""
|
||
INSERT INTO objective_complex_mapping
|
||
(objective_complex_name, domrf_obj_id, objective_group,
|
||
match_method, match_score, is_reviewed)
|
||
VALUES (
|
||
CAST(:name AS text),
|
||
CAST(:obj_id AS bigint),
|
||
CAST(:group AS text),
|
||
CAST(:method AS text),
|
||
CAST(:score AS numeric),
|
||
CAST(:reviewed AS boolean)
|
||
)
|
||
ON CONFLICT (objective_complex_name, objective_group) DO NOTHING
|
||
"""
|
||
),
|
||
{
|
||
"name": c.objective_project_name,
|
||
"obj_id": c.domrf_obj_id,
|
||
"group": "Екатеринбург",
|
||
"method": match_method,
|
||
"score": c.similarity_score,
|
||
"reviewed": False,
|
||
},
|
||
)
|
||
if result.rowcount > 0:
|
||
inserted += 1
|
||
else:
|
||
skipped += 1
|
||
except Exception as e:
|
||
logger.warning(
|
||
"Insert failed для %s ↔ %s: %s",
|
||
c.domrf_comm_name,
|
||
c.objective_project_name,
|
||
e,
|
||
)
|
||
skipped += 1
|
||
|
||
db.commit()
|
||
logger.info(
|
||
"Backfill complete: auto_accepted=%d skipped=%d review_queue=%d",
|
||
inserted,
|
||
skipped,
|
||
len(review),
|
||
)
|
||
return {"auto_accepted": inserted, "review_queue": len(review), "skipped": skipped}
|
||
|
||
|
||
def trigger_mv_refresh(db: Session) -> int:
|
||
"""REFRESH mv_layout_velocity после backfill.
|
||
|
||
Вызывает существующий helper из layout_velocity_refresh.
|
||
Передаём concurrently=True (MV уже заполнен).
|
||
"""
|
||
from app.services.site_finder.layout_velocity_refresh import refresh_layout_velocity
|
||
|
||
return refresh_layout_velocity(db, concurrently=True)
|