4c68645e4f
Add fuzzy-match ETL service that maps unmapped DOM.РФ EKB complexes to Objective project names using pg_trgm similarity. Exposes admin endpoint POST /api/v1/admin/etl/objective-backfill with dry_run + refresh_mv flags. Phase 1 scope: auto-accept matches >= 0.85, review_queue 0.60-0.84 (UI TBD). Expected coverage lift: 114 → ~260 mapped objects, mv_layout_velocity 3% → ~20%. Closes #203
70 lines
2.4 KiB
Python
70 lines
2.4 KiB
Python
"""Admin endpoints для ETL operations (#203).
|
||
|
||
POST /api/v1/admin/etl/objective-backfill
|
||
Запустить fuzzy-match backfill objective_complex_mapping.
|
||
Опциональный REFRESH mv_layout_velocity после успешного backfill.
|
||
|
||
Header: X-Admin-Token: <SCRAPE_ADMIN_TOKEN>
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from typing import Annotated
|
||
|
||
from fastapi import APIRouter, Depends, Query
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.core.db import get_db
|
||
from app.core.deps import AdminTokenAuth
|
||
from app.services.etl.objective_backfill import (
|
||
REVIEW_THRESHOLD,
|
||
auto_apply_matches,
|
||
find_match_candidates,
|
||
trigger_mv_refresh,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
router = APIRouter()
|
||
|
||
|
||
@router.post("/objective-backfill")
|
||
def run_objective_backfill(
|
||
db: Annotated[Session, Depends(get_db)],
|
||
_: AdminTokenAuth,
|
||
dry_run: Annotated[bool, Query(description="Preview без insertions")] = False,
|
||
refresh_mv: Annotated[
|
||
bool, Query(description="REFRESH mv_layout_velocity после backfill")
|
||
] = True,
|
||
) -> dict[str, int]:
|
||
"""Запустить backfill objective_complex_mapping + опционально REFRESH MV.
|
||
|
||
Ищет DOM.РФ комплексы (is_ekb=true) без mapping и применяет fuzzy match
|
||
к project_name из objective_corpus_room_month через pg_trgm similarity.
|
||
|
||
- score >= 0.85 (AUTO_ACCEPT_THRESHOLD): auto-insert с match_method='fuzzy_trgm'
|
||
- score >= 0.6 (REVIEW_THRESHOLD) и < 0.85: только в счётчик review_queue
|
||
(Phase 2 — UI для ручного review)
|
||
|
||
Returns dict:
|
||
auto_accepted: сколько строк вставлено
|
||
review_queue: сколько кандидатов ниже порога auto-accept
|
||
skipped: ON CONFLICT + ошибки
|
||
mv_rows_after_refresh: строк в MV после REFRESH (0 если refresh_mv=False)
|
||
"""
|
||
candidates = find_match_candidates(db, only_unmapped=True)
|
||
logger.info(
|
||
"Backfill candidates found: %d (score >= %.2f)",
|
||
len(candidates),
|
||
REVIEW_THRESHOLD,
|
||
)
|
||
|
||
result = auto_apply_matches(db, candidates, dry_run=dry_run)
|
||
|
||
mv_rows = 0
|
||
if refresh_mv and not dry_run and result["auto_accepted"] > 0:
|
||
mv_rows = trigger_mv_refresh(db)
|
||
logger.info("mv_layout_velocity refreshed after backfill: %d rows", mv_rows)
|
||
|
||
result["mv_rows_after_refresh"] = mv_rows
|
||
return result
|