From 029dcb4cb15148d6259be6e2d0a557b47693fde0 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Fri, 15 May 2026 13:38:39 +0300 Subject: [PATCH] feat(admin): cadastre bulk harvest UI at /admin/scrape/cadastre (#168 PR4/5) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add admin page for triggering and monitoring cadastre_jobs: - Trigger form: pilot (50) / ekb_full (2408) / manual_list scopes - Live jobs table 5s auto-refresh (TanStack Query) - Per-row expand: phase_state JSON, req/s rate, WAF count, error - Cancel button for queued/running, Resume for failed/cancelled/paused - Heartbeat staleness: red "STALE" if running and gap > 10min - WAF count highlighted red+⚠ when > 5 - Progress bar from targets_done/targets_total - New tab in admin layout - Token in localStorage same as geo page --- frontend/src/app/admin/layout.tsx | 1 + .../src/app/admin/scrape/cadastre/page.tsx | 969 ++++++++++++++++++ 2 files changed, 970 insertions(+) create mode 100644 frontend/src/app/admin/scrape/cadastre/page.tsx diff --git a/frontend/src/app/admin/layout.tsx b/frontend/src/app/admin/layout.tsx index 8f4bad2c..b2eaa04d 100644 --- a/frontend/src/app/admin/layout.tsx +++ b/frontend/src/app/admin/layout.tsx @@ -7,6 +7,7 @@ const TABS = [ { href: "/admin/scrape/all", label: "📊 Все скраперы (сводно)" }, { href: "/admin/scrape", label: "Скрапер DOM.РФ" }, { href: "/admin/scrape/geo", label: "🗺 NSPD geo (rosreestr2coord)" }, + { href: "/admin/scrape/cadastre", label: "🏗 Cadastre bulk harvest" }, { href: "/admin/scrape/objective", label: "ETL Objective" }, { href: "/admin/leads", label: "Лиды PRINZIP" }, ]; diff --git a/frontend/src/app/admin/scrape/cadastre/page.tsx b/frontend/src/app/admin/scrape/cadastre/page.tsx new file mode 100644 index 00000000..95958980 --- /dev/null +++ b/frontend/src/app/admin/scrape/cadastre/page.tsx @@ -0,0 +1,969 @@ +"use client"; + +import { useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { apiFetch } from "@/lib/api"; +import { cardStyle, inputStyle, labelStyle, td, th } from "@/lib/adminStyles"; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +interface PhaseState { + phase?: string; + parcels_meta?: number; + parcels_fetched?: number; + buildings_meta?: number; + buildings_fetched?: number; + retries?: number; +} + +interface CadastreJob { + job_id: number; + name: string | null; + job_kind: string; + scope: string; + status: + | "queued" + | "running" + | "done" + | "failed" + | "zombie" + | "cancelled" + | "paused"; + triggered_by: string | null; + rate_ms: number; + created_at: string | null; + started_at: string | null; + finished_at: string | null; + heartbeat_at: string | null; + targets_total: number; + targets_done: number; + targets_failed: number; + targets_skipped: number; + requests_count: number; + waf_blocked_count: number; + error: string | null; + phase_state: PhaseState | null; + progress_pct?: number; +} + +interface CreateCadastreJobRequest { + scope: "pilot" | "ekb_full" | "manual_list"; + name?: string; + quarters?: string[]; + limit?: number; +} + +interface CreateCadastreJobResponse { + job_id: number; + scope: string; + targets_total: number; + estimate_minutes: number; +} + +type ScopeOption = "pilot" | "ekb_full" | "manual_list"; + +// --------------------------------------------------------------------------- +// Constants +// --------------------------------------------------------------------------- + +const STATUS_COLOR: Record = { + queued: "#6b7280", + running: "#1d4ed8", + done: "#16a34a", + failed: "#b3261e", + zombie: "#dc2626", + cancelled: "#9333ea", + paused: "#a16207", +}; + +const SCOPE_LABEL: Record = { + pilot: "Pilot (50 кварталов ЕКБ)", + ekb_full: "Full ЕКБ (2 408 кварталов)", + manual_list: "Manual list", +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function formatIso(s: string | null): string { + if (!s) return "—"; + return new Date(s).toLocaleString("ru-RU"); +} + +function diffSecs(a: string | null, b: string | null): string { + if (!a || !b) return "—"; + const dt = (new Date(b).getTime() - new Date(a).getTime()) / 1000; + if (dt < 60) return `${dt.toFixed(0)} с`; + if (dt < 3600) return `${(dt / 60).toFixed(1)} мин`; + return `${(dt / 3600).toFixed(1)} ч`; +} + +function calcProgress(job: CadastreJob): number { + if (job.progress_pct !== undefined) return job.progress_pct; + if (job.targets_total === 0) return 0; + return (job.targets_done / job.targets_total) * 100; +} + +function reqPerSec(job: CadastreJob): string { + if (!job.started_at || job.requests_count === 0) return "—"; + const endTs = job.heartbeat_at ?? job.finished_at; + if (!endTs) return "—"; + const elapsed = + (new Date(endTs).getTime() - new Date(job.started_at).getTime()) / 1000; + if (elapsed <= 0) return "—"; + return (job.requests_count / elapsed).toFixed(2); +} + +// --------------------------------------------------------------------------- +// Query keys +// --------------------------------------------------------------------------- + +const jobsKey = (token: string) => ["admin", "cadastre", "jobs", token]; +const jobKey = (token: string, id: number) => [ + "admin", + "cadastre", + "job", + id, + token, +]; + +// --------------------------------------------------------------------------- +// Component +// --------------------------------------------------------------------------- + +export default function CadastreScrapeAdminPage() { + const [token, setToken] = useState(() => + typeof window === "undefined" + ? "" + : (localStorage.getItem("admin_token") ?? ""), + ); + const [scope, setScope] = useState("pilot"); + const [name, setName] = useState(""); + const [quartersText, setQuartersText] = useState(""); + const [limit, setLimit] = useState(""); + const [lastResp, setLastResp] = useState( + null, + ); + const [expandedId, setExpandedId] = useState(null); + + const queryClient = useQueryClient(); + + const saveToken = (v: string) => { + setToken(v); + if (typeof window !== "undefined") { + localStorage.setItem("admin_token", v); + } + }; + + // ------------------------------------------------------------------------- + // Queries + // ------------------------------------------------------------------------- + + const jobs = useQuery({ + queryKey: jobsKey(token), + queryFn: () => + apiFetch("/api/v1/admin/cadastre/jobs?limit=30", { + headers: { "X-Admin-Token": token }, + }), + enabled: !!token, + refetchInterval: 5000, + }); + + // Per-job detail (only fetched when row is expanded) + const expandedJob = useQuery({ + queryKey: jobKey(token, expandedId ?? 0), + queryFn: () => + apiFetch(`/api/v1/admin/cadastre/jobs/${expandedId}`, { + headers: { "X-Admin-Token": token }, + }), + enabled: !!token && expandedId !== null, + refetchInterval: expandedId !== null ? 5000 : false, + }); + + // ------------------------------------------------------------------------- + // Mutations + // ------------------------------------------------------------------------- + + const createJob = useMutation({ + mutationFn: () => { + const body: CreateCadastreJobRequest = { scope }; + if (name.trim()) body.name = name.trim(); + if (scope === "manual_list") { + body.quarters = quartersText + .split(/[\n,]+/) + .map((s) => s.trim()) + .filter(Boolean); + } + const limitNum = parseInt(limit, 10); + if (!isNaN(limitNum) && limitNum > 0) body.limit = limitNum; + return apiFetch( + "/api/v1/admin/cadastre/jobs", + { + method: "POST", + headers: { + "X-Admin-Token": token, + "Content-Type": "application/json", + }, + body: JSON.stringify(body), + }, + ); + }, + onSuccess: (data) => { + setLastResp(data); + void queryClient.invalidateQueries({ queryKey: jobsKey(token) }); + }, + }); + + const cancelJob = useMutation({ + mutationFn: (id: number) => + apiFetch<{ cancelled: boolean }>( + `/api/v1/admin/cadastre/jobs/${id}/cancel`, + { method: "POST", headers: { "X-Admin-Token": token } }, + ), + onSuccess: (_data, id) => { + void queryClient.invalidateQueries({ queryKey: jobKey(token, id) }); + void queryClient.invalidateQueries({ queryKey: jobsKey(token) }); + }, + }); + + const resumeJob = useMutation({ + mutationFn: (id: number) => + apiFetch<{ resumed: boolean }>( + `/api/v1/admin/cadastre/jobs/${id}/resume`, + { method: "POST", headers: { "X-Admin-Token": token } }, + ), + onSuccess: (_data, id) => { + void queryClient.invalidateQueries({ queryKey: jobKey(token, id) }); + void queryClient.invalidateQueries({ queryKey: jobsKey(token) }); + }, + }); + + // ------------------------------------------------------------------------- + // Handlers + // ------------------------------------------------------------------------- + + const handleSubmit = (e: React.FormEvent) => { + e.preventDefault(); + if (!token) return; + if (scope === "manual_list" && !quartersText.trim()) { + alert("Для Manual list укажите хотя бы один квартал"); + return; + } + createJob.mutate(); + }; + + const toggleExpand = (id: number) => { + setExpandedId((prev) => (prev === id ? null : id)); + }; + + // ------------------------------------------------------------------------- + // Render helpers + // ------------------------------------------------------------------------- + + const renderProgressBar = (job: CadastreJob) => { + const pct = calcProgress(job); + const color = STATUS_COLOR[job.status] ?? "#6b7280"; + return ( +
+ {pct.toFixed(1)}% +
+
+
+
+ ); + }; + + const renderHeartbeat = (job: CadastreJob) => { + if (!job.heartbeat_at) return —; + const lagMs = Date.now() - new Date(job.heartbeat_at).getTime(); + const lagMin = lagMs / 60000; + const stale = job.status === "running" && lagMin > 10; + const lagStr = + lagMs < 60000 + ? `${Math.round(lagMs / 1000)} с` + : `${lagMin.toFixed(1)} мин`; + return ( + + {stale ? `STALE (${lagStr})` : lagStr} + + ); + }; + + const renderStatusBadge = (job: CadastreJob) => { + const color = STATUS_COLOR[job.status] ?? "#6b7280"; + const heartbeatLag = job.heartbeat_at + ? (Date.now() - new Date(job.heartbeat_at).getTime()) / 60000 + : null; + const isZombie = + job.status === "running" && heartbeatLag !== null && heartbeatLag > 10; + return ( + + {isZombie ? "zombie" : job.status} + + ); + }; + + const renderActions = (job: CadastreJob) => ( +
+ {(job.status === "running" || job.status === "queued") && ( + + )} + {(job.status === "failed" || + job.status === "cancelled" || + job.status === "paused") && + job.phase_state?.phase !== "done" && ( + + )} +
+ ); + + const renderExpandedPanel = (job: CadastreJob | undefined) => { + if (!job) { + return ( + + + Загрузка деталей... + + + ); + } + return ( + + +
+ {/* Phase state */} +
+
+ PHASE STATE +
+ {job.phase_state ? ( + 
+                  {JSON.stringify(job.phase_state, null, 2)}
+
+ ) : ( + — + )} +
+ + {/* Stats */} +
+
+ СТАТИСТИКА +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Запросов + + {job.requests_count.toLocaleString("ru-RU")} +
+ Req/s + + {reqPerSec(job)} +
+ WAF blocked + 5 ? "#dc2626" : "inherit", + fontWeight: job.waf_blocked_count > 5 ? 700 : 400, + }} + > + {job.waf_blocked_count} + {job.waf_blocked_count > 5 && " ⚠ HIGH"} +
+ Skipped + + {job.targets_skipped} +
+ Rate ms + + {job.rate_ms} +
+ Started + + {formatIso(job.started_at)} +
+ Finished + + {formatIso(job.finished_at)} +
+ Triggered by + + {job.triggered_by ?? "—"} +
+
+ + {/* Error */} + {job.error && ( +
+
+ ERROR +
+ 
+                  {job.error}
+
+
+ )} +
+ + + ); + }; + + // ------------------------------------------------------------------------- + // Render + // ------------------------------------------------------------------------- + + return ( + <> +

+ Cadastre bulk harvest — ручной запуск +

+

+ Bulk-загрузка кадастровых данных (участки + здания) из NSPD. State + хранится в БД (cadastre_jobs +{" "} + cadastre_targets) — при redeploy job продолжается с + незавершённых targets. +

+ + {/* Admin Token */} +
+ +
+ + {/* Create job form */} +
+

Запуск bulk harvest

+
+
+ + + {/* Scope radio buttons */} +
+ Scope +
+ {(["pilot", "ekb_full", "manual_list"] as ScopeOption[]).map( + (s) => ( + + ), + )} +
+
+ + {/* Manual quarters textarea */} + {scope === "manual_list" && ( +