2694e3180b
- Add AUTO_ACCEPT_THRESHOLD_V2 = 0.80 constant to objective_backfill.py - Add min_threshold param to find_match_candidates() (default REVIEW_THRESHOLD=0.6) - Add match_method param to auto_apply_matches() (default 'fuzzy_trgm') - Add ?v2=true query param to POST /api/v1/admin/etl/objective-backfill - v2 mode: threshold=0.80, method='fuzzy_v2', search from 0.80 (not 0.60) - Return type widened to dict[str,object] to include threshold_used + match_method_used DB audit: 1068 unmapped EKB objs; v1 adds ~133 rows, v2 adds ~47 more. Expected coverage: 8.5% → ~20% after sequential v1+v2 runs on prod. Run: POST /objective-backfill (v1) then POST /objective-backfill?v2=true (v2). Part of task #44 Part A, epic #271.
128 lines
5 KiB
Python
128 lines
5 KiB
Python
"""Admin endpoints для ETL operations (#203, #44).
|
||
|
||
POST /api/v1/admin/etl/objective-backfill
|
||
Запустить fuzzy-match backfill objective_complex_mapping.
|
||
Поддерживает два режима:
|
||
- v1 (default): threshold=0.85, match_method='fuzzy_trgm'
|
||
- v2: threshold=0.80, match_method='fuzzy_v2' — для coverage 8.5%→17%
|
||
|
||
POST /api/v1/admin/etl/nspd-denorm-backfill
|
||
Запустить backfill nspd_parcels/nspd_buildings из всех nspd_quarter_dumps.
|
||
|
||
Header: X-Admin-Token: <SCRAPE_ADMIN_TOKEN>
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from typing import Annotated
|
||
|
||
from fastapi import APIRouter, Depends, Query
|
||
from sqlalchemy.orm import Session
|
||
|
||
from app.core.db import get_db
|
||
from app.core.deps import AdminTokenAuth
|
||
from app.services.etl.objective_backfill import (
|
||
AUTO_ACCEPT_THRESHOLD,
|
||
AUTO_ACCEPT_THRESHOLD_V2,
|
||
REVIEW_THRESHOLD,
|
||
auto_apply_matches,
|
||
find_match_candidates,
|
||
trigger_mv_refresh,
|
||
)
|
||
|
||
logger = logging.getLogger(__name__)
|
||
router = APIRouter()
|
||
|
||
|
||
@router.post("/objective-backfill")
|
||
def run_objective_backfill(
|
||
db: Annotated[Session, Depends(get_db)],
|
||
_: AdminTokenAuth,
|
||
dry_run: Annotated[bool, Query(description="Preview без insertions")] = False,
|
||
refresh_mv: Annotated[
|
||
bool, Query(description="REFRESH mv_layout_velocity после backfill")
|
||
] = True,
|
||
v2: Annotated[
|
||
bool,
|
||
Query(
|
||
description=(
|
||
"v2 mode: threshold=0.80, match_method='fuzzy_v2'. "
|
||
"Запускать после v1 run — только для unmapped объектов."
|
||
)
|
||
),
|
||
] = False,
|
||
) -> dict[str, object]:
|
||
"""Запустить backfill objective_complex_mapping + опционально REFRESH MV.
|
||
|
||
Ищет DOM.РФ комплексы (is_ekb=true) без mapping и применяет fuzzy match
|
||
к project_name из objective_corpus_room_month через pg_trgm similarity.
|
||
|
||
Режим v1 (default, ?v2=false):
|
||
- score >= 0.85 (AUTO_ACCEPT_THRESHOLD): auto-insert, match_method='fuzzy_trgm'
|
||
- score >= 0.6 (REVIEW_THRESHOLD) и < 0.85: только счётчик review_queue
|
||
|
||
Режим v2 (?v2=true, task #44 coverage expansion):
|
||
- score >= 0.80 (AUTO_ACCEPT_THRESHOLD_V2): auto-insert, match_method='fuzzy_v2'
|
||
- is_reviewed=false — требует ручной проверки (false positives вероятны)
|
||
- Целевой прирост: +~47-80 строк, coverage 8.5% → ~17%
|
||
|
||
Returns dict:
|
||
auto_accepted: сколько строк вставлено
|
||
review_queue: сколько кандидатов ниже порога auto-accept
|
||
skipped: ON CONFLICT + ошибки
|
||
mv_rows_after_refresh: строк в MV после REFRESH (0 если refresh_mv=False)
|
||
threshold_used: фактический порог (float)
|
||
match_method_used: match_method в БД (str)
|
||
"""
|
||
threshold = AUTO_ACCEPT_THRESHOLD_V2 if v2 else AUTO_ACCEPT_THRESHOLD
|
||
method = "fuzzy_v2" if v2 else "fuzzy_trgm"
|
||
# v2 ищет кандидатов начиная с threshold (не от REVIEW_THRESHOLD)
|
||
search_min = threshold if v2 else REVIEW_THRESHOLD
|
||
|
||
candidates = find_match_candidates(db, only_unmapped=True, min_threshold=search_min)
|
||
logger.info(
|
||
"Backfill candidates found: %d (score >= %.2f, method=%s)",
|
||
len(candidates),
|
||
search_min,
|
||
method,
|
||
)
|
||
|
||
result: dict[str, object] = dict(
|
||
auto_apply_matches(
|
||
db, candidates, threshold=threshold, match_method=method, dry_run=dry_run
|
||
)
|
||
)
|
||
|
||
mv_rows = 0
|
||
if refresh_mv and not dry_run and result.get("auto_accepted", 0):
|
||
mv_rows = trigger_mv_refresh(db)
|
||
logger.info("mv_layout_velocity refreshed after backfill: %d rows", mv_rows)
|
||
|
||
result["mv_rows_after_refresh"] = mv_rows
|
||
result["threshold_used"] = threshold
|
||
result["match_method_used"] = method
|
||
return result
|
||
|
||
|
||
@router.post("/nspd-denorm-backfill")
|
||
def run_nspd_denorm_backfill(
|
||
_: AdminTokenAuth,
|
||
limit: Annotated[
|
||
int | None,
|
||
Query(description="Максимум кварталов для обработки (None = все)"),
|
||
] = None,
|
||
) -> dict[str, object]:
|
||
"""Запустить Celery backfill: denormalize nspd_quarter_dumps → nspd_parcels/nspd_buildings.
|
||
|
||
Задача идемпотентна (ON CONFLICT DO UPDATE). Безопасно запускать повторно.
|
||
Возвращает Celery task_id — статус через /flower или celery inspect.
|
||
|
||
Args:
|
||
limit: если задан — обработать только первые N кварталов ORDER BY quarter_cad.
|
||
"""
|
||
from app.workers.tasks.nspd_denorm_backfill import backfill_all_dumps
|
||
|
||
task = backfill_all_dumps.apply_async(kwargs={"limit": limit})
|
||
logger.info("nspd-denorm-backfill enqueued: task_id=%s limit=%s", task.id, limit)
|
||
return {"task_id": task.id, "status": "enqueued", "limit": limit}
|