gendesign/frontend/src/app/admin/scrape/cadastre/page.tsx
lekss361 ced237804d
feat(admin): cadastre bulk harvest UI at /admin/scrape/cadastre (#168 PR4/5) (#172)
Add admin page for triggering and monitoring cadastre_jobs:
- Trigger form: pilot (50) / ekb_full (2408) / manual_list scopes
- Live jobs table 5s auto-refresh (TanStack Query)
- Per-row expand: phase_state JSON, req/s rate, WAF count, error
- Cancel button for queued/running, Resume for failed/cancelled/paused
- Heartbeat staleness: red "STALE" if running and gap > 10min
- WAF count highlighted red+⚠ when > 5
- Progress bar from targets_done/targets_total
- New tab in admin layout
- Token in localStorage same as geo page

Co-authored-by: lekss361 <claudestars@proton.me>
2026-05-15 13:46:07 +03:00

969 lines
30 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"use client";
import { useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { apiFetch } from "@/lib/api";
import { cardStyle, inputStyle, labelStyle, td, th } from "@/lib/adminStyles";
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
interface PhaseState {
phase?: string;
parcels_meta?: number;
parcels_fetched?: number;
buildings_meta?: number;
buildings_fetched?: number;
retries?: number;
}
interface CadastreJob {
job_id: number;
name: string | null;
job_kind: string;
scope: string;
status:
| "queued"
| "running"
| "done"
| "failed"
| "zombie"
| "cancelled"
| "paused";
triggered_by: string | null;
rate_ms: number;
created_at: string | null;
started_at: string | null;
finished_at: string | null;
heartbeat_at: string | null;
targets_total: number;
targets_done: number;
targets_failed: number;
targets_skipped: number;
requests_count: number;
waf_blocked_count: number;
error: string | null;
phase_state: PhaseState | null;
progress_pct?: number;
}
interface CreateCadastreJobRequest {
scope: "pilot" | "ekb_full" | "manual_list";
name?: string;
quarters?: string[];
limit?: number;
}
interface CreateCadastreJobResponse {
job_id: number;
scope: string;
targets_total: number;
estimate_minutes: number;
}
type ScopeOption = "pilot" | "ekb_full" | "manual_list";
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
const STATUS_COLOR: Record<string, string> = {
queued: "#6b7280",
running: "#1d4ed8",
done: "#16a34a",
failed: "#b3261e",
zombie: "#dc2626",
cancelled: "#9333ea",
paused: "#a16207",
};
const SCOPE_LABEL: Record<ScopeOption, string> = {
pilot: "Pilot (50 кварталов ЕКБ)",
ekb_full: "Full ЕКБ (2 408 кварталов)",
manual_list: "Manual list",
};
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function formatIso(s: string | null): string {
if (!s) return "—";
return new Date(s).toLocaleString("ru-RU");
}
function diffSecs(a: string | null, b: string | null): string {
if (!a || !b) return "—";
const dt = (new Date(b).getTime() - new Date(a).getTime()) / 1000;
if (dt < 60) return `${dt.toFixed(0)} с`;
if (dt < 3600) return `${(dt / 60).toFixed(1)} мин`;
return `${(dt / 3600).toFixed(1)} ч`;
}
function calcProgress(job: CadastreJob): number {
if (job.progress_pct !== undefined) return job.progress_pct;
if (job.targets_total === 0) return 0;
return (job.targets_done / job.targets_total) * 100;
}
function reqPerSec(job: CadastreJob): string {
if (!job.started_at || job.requests_count === 0) return "—";
const endTs = job.heartbeat_at ?? job.finished_at;
if (!endTs) return "—";
const elapsed =
(new Date(endTs).getTime() - new Date(job.started_at).getTime()) / 1000;
if (elapsed <= 0) return "—";
return (job.requests_count / elapsed).toFixed(2);
}
// ---------------------------------------------------------------------------
// Query keys
// ---------------------------------------------------------------------------
const jobsKey = (token: string) => ["admin", "cadastre", "jobs", token];
const jobKey = (token: string, id: number) => [
"admin",
"cadastre",
"job",
id,
token,
];
// ---------------------------------------------------------------------------
// Component
// ---------------------------------------------------------------------------
export default function CadastreScrapeAdminPage() {
const [token, setToken] = useState<string>(() =>
typeof window === "undefined"
? ""
: (localStorage.getItem("admin_token") ?? ""),
);
const [scope, setScope] = useState<ScopeOption>("pilot");
const [name, setName] = useState("");
const [quartersText, setQuartersText] = useState("");
const [limit, setLimit] = useState<string>("");
const [lastResp, setLastResp] = useState<CreateCadastreJobResponse | null>(
null,
);
const [expandedId, setExpandedId] = useState<number | null>(null);
const queryClient = useQueryClient();
const saveToken = (v: string) => {
setToken(v);
if (typeof window !== "undefined") {
localStorage.setItem("admin_token", v);
}
};
// -------------------------------------------------------------------------
// Queries
// -------------------------------------------------------------------------
const jobs = useQuery({
queryKey: jobsKey(token),
queryFn: () =>
apiFetch<CadastreJob[]>("/api/v1/admin/cadastre/jobs?limit=30", {
headers: { "X-Admin-Token": token },
}),
enabled: !!token,
refetchInterval: 5000,
});
// Per-job detail (only fetched when row is expanded)
const expandedJob = useQuery({
queryKey: jobKey(token, expandedId ?? 0),
queryFn: () =>
apiFetch<CadastreJob>(`/api/v1/admin/cadastre/jobs/${expandedId}`, {
headers: { "X-Admin-Token": token },
}),
enabled: !!token && expandedId !== null,
refetchInterval: expandedId !== null ? 5000 : false,
});
// -------------------------------------------------------------------------
// Mutations
// -------------------------------------------------------------------------
const createJob = useMutation({
mutationFn: () => {
const body: CreateCadastreJobRequest = { scope };
if (name.trim()) body.name = name.trim();
if (scope === "manual_list") {
body.quarters = quartersText
.split(/[\n,]+/)
.map((s) => s.trim())
.filter(Boolean);
}
const limitNum = parseInt(limit, 10);
if (!isNaN(limitNum) && limitNum > 0) body.limit = limitNum;
return apiFetch<CreateCadastreJobResponse>(
"/api/v1/admin/cadastre/jobs",
{
method: "POST",
headers: {
"X-Admin-Token": token,
"Content-Type": "application/json",
},
body: JSON.stringify(body),
},
);
},
onSuccess: (data) => {
setLastResp(data);
void queryClient.invalidateQueries({ queryKey: jobsKey(token) });
},
});
const cancelJob = useMutation({
mutationFn: (id: number) =>
apiFetch<{ cancelled: boolean }>(
`/api/v1/admin/cadastre/jobs/${id}/cancel`,
{ method: "POST", headers: { "X-Admin-Token": token } },
),
onSuccess: (_data, id) => {
void queryClient.invalidateQueries({ queryKey: jobKey(token, id) });
void queryClient.invalidateQueries({ queryKey: jobsKey(token) });
},
});
const resumeJob = useMutation({
mutationFn: (id: number) =>
apiFetch<{ resumed: boolean }>(
`/api/v1/admin/cadastre/jobs/${id}/resume`,
{ method: "POST", headers: { "X-Admin-Token": token } },
),
onSuccess: (_data, id) => {
void queryClient.invalidateQueries({ queryKey: jobKey(token, id) });
void queryClient.invalidateQueries({ queryKey: jobsKey(token) });
},
});
// -------------------------------------------------------------------------
// Handlers
// -------------------------------------------------------------------------
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault();
if (!token) return;
if (scope === "manual_list" && !quartersText.trim()) {
alert("Для Manual list укажите хотя бы один квартал");
return;
}
createJob.mutate();
};
const toggleExpand = (id: number) => {
setExpandedId((prev) => (prev === id ? null : id));
};
// -------------------------------------------------------------------------
// Render helpers
// -------------------------------------------------------------------------
const renderProgressBar = (job: CadastreJob) => {
const pct = calcProgress(job);
const color = STATUS_COLOR[job.status] ?? "#6b7280";
return (
<div
style={{
display: "flex",
flexDirection: "column",
gap: 2,
alignItems: "flex-end",
}}
>
<strong>{pct.toFixed(1)}%</strong>
<div
style={{
width: 80,
height: 6,
background: "#e5e7eb",
borderRadius: 3,
overflow: "hidden",
}}
>
<div
style={{
width: `${Math.min(pct, 100)}%`,
height: "100%",
background: color,
}}
/>
</div>
</div>
);
};
const renderHeartbeat = (job: CadastreJob) => {
if (!job.heartbeat_at) return <span style={{ color: "#9ca3af" }}></span>;
const lagMs = Date.now() - new Date(job.heartbeat_at).getTime();
const lagMin = lagMs / 60000;
const stale = job.status === "running" && lagMin > 10;
const lagStr =
lagMs < 60000
? `${Math.round(lagMs / 1000)} с`
: `${lagMin.toFixed(1)} мин`;
return (
<span style={{ color: stale ? "#dc2626" : "inherit", fontSize: 11 }}>
{stale ? `STALE (${lagStr})` : lagStr}
</span>
);
};
const renderStatusBadge = (job: CadastreJob) => {
const color = STATUS_COLOR[job.status] ?? "#6b7280";
const heartbeatLag = job.heartbeat_at
? (Date.now() - new Date(job.heartbeat_at).getTime()) / 60000
: null;
const isZombie =
job.status === "running" && heartbeatLag !== null && heartbeatLag > 10;
return (
<span
style={{
display: "inline-block",
padding: "2px 8px",
borderRadius: 4,
background: color + "22",
color,
fontWeight: 500,
}}
>
{isZombie ? "zombie" : job.status}
</span>
);
};
const renderActions = (job: CadastreJob) => (
<div style={{ display: "flex", gap: 4 }}>
{(job.status === "running" || job.status === "queued") && (
<button
type="button"
onClick={() => cancelJob.mutate(job.job_id)}
disabled={cancelJob.isPending}
style={{ ...secondaryBtn, padding: "4px 8px", fontSize: 11 }}
>
Cancel
</button>
)}
{(job.status === "failed" ||
job.status === "cancelled" ||
job.status === "paused") &&
job.phase_state?.phase !== "done" && (
<button
type="button"
onClick={() => resumeJob.mutate(job.job_id)}
disabled={resumeJob.isPending}
style={{
...secondaryBtn,
padding: "4px 8px",
fontSize: 11,
background: "#dbeafe",
color: "#1d4ed8",
}}
>
Resume
</button>
)}
</div>
);
const renderExpandedPanel = (job: CadastreJob | undefined) => {
if (!job) {
return (
<tr>
<td
colSpan={10}
style={{
...td,
background: "#f9fafb",
fontSize: 12,
color: "#5b6066",
}}
>
Загрузка деталей...
</td>
</tr>
);
}
return (
<tr>
<td
colSpan={10}
style={{
...td,
background: "#f9fafb",
borderBottom: "2px solid #e6e8ec",
}}
>
<div
style={{
display: "grid",
gridTemplateColumns: "repeat(auto-fit, minmax(220px, 1fr))",
gap: 16,
fontSize: 13,
}}
>
{/* Phase state */}
<div>
<div
style={{
fontWeight: 600,
marginBottom: 6,
fontSize: 12,
color: "#5b6066",
}}
>
PHASE STATE
</div>
{job.phase_state ? (
<pre
style={{
margin: 0,
fontFamily: "monospace",
fontSize: 11,
background: "#1e293b",
color: "#e2e8f0",
padding: "8px 10px",
borderRadius: 6,
overflowX: "auto",
}}
>
{JSON.stringify(job.phase_state, null, 2)}
</pre>
) : (
<span style={{ color: "#9ca3af", fontSize: 12 }}></span>
)}
</div>
{/* Stats */}
<div>
<div
style={{
fontWeight: 600,
marginBottom: 6,
fontSize: 12,
color: "#5b6066",
}}
>
СТАТИСТИКА
</div>
<table style={{ fontSize: 12, borderCollapse: "collapse" }}>
<tbody>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Запросов
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
}}
>
{job.requests_count.toLocaleString("ru-RU")}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Req/s
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
}}
>
{reqPerSec(job)}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
WAF blocked
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
color:
job.waf_blocked_count > 5 ? "#dc2626" : "inherit",
fontWeight: job.waf_blocked_count > 5 ? 700 : 400,
}}
>
{job.waf_blocked_count}
{job.waf_blocked_count > 5 && " ⚠ HIGH"}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Skipped
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
}}
>
{job.targets_skipped}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Rate ms
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
}}
>
{job.rate_ms}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Started
</td>
<td style={{ ...td, padding: "3px 0", fontSize: 11 }}>
{formatIso(job.started_at)}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Finished
</td>
<td style={{ ...td, padding: "3px 0", fontSize: 11 }}>
{formatIso(job.finished_at)}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Triggered by
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
}}
>
{job.triggered_by ?? "—"}
</td>
</tr>
</tbody>
</table>
</div>
{/* Error */}
{job.error && (
<div>
<div
style={{
fontWeight: 600,
marginBottom: 6,
fontSize: 12,
color: "#dc2626",
}}
>
ERROR
</div>
<pre
style={{
margin: 0,
fontFamily: "monospace",
fontSize: 11,
background: "#fef2f2",
color: "#991b1b",
padding: "8px 10px",
borderRadius: 6,
overflowX: "auto",
whiteSpace: "pre-wrap",
wordBreak: "break-all",
}}
>
{job.error}
</pre>
</div>
)}
</div>
</td>
</tr>
);
};
// -------------------------------------------------------------------------
// Render
// -------------------------------------------------------------------------
return (
<>
<h2 style={{ marginTop: 0, fontSize: 18 }}>
Cadastre bulk harvest ручной запуск
</h2>
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 4 }}>
Bulk-загрузка кадастровых данных (участки + здания) из NSPD. State
хранится в БД (<code>cadastre_jobs</code> +{" "}
<code>cadastre_targets</code>) при redeploy job продолжается с
незавершённых targets.
</p>
{/* Admin Token */}
<section style={cardStyle}>
<label style={{ display: "block" }}>
<span style={labelStyle}>Admin Token</span>
<input
type="password"
value={token}
onChange={(e) => saveToken(e.target.value)}
placeholder="SCRAPE_ADMIN_TOKEN"
style={inputStyle}
/>
</label>
</section>
{/* Create job form */}
<section style={{ ...cardStyle, marginTop: 16 }}>
<h3 style={sectionTitle}>Запуск bulk harvest</h3>
<form onSubmit={handleSubmit}>
<div style={{ display: "grid", gap: 12 }}>
<label>
<span style={labelStyle}>Имя задачи (опционально)</span>
<input
type="text"
value={name}
onChange={(e) => setName(e.target.value)}
placeholder={`pilot_${new Date().toISOString().slice(0, 10)}`}
style={inputStyle}
/>
</label>
{/* Scope radio buttons */}
<div>
<span style={labelStyle}>Scope</span>
<div style={{ display: "flex", flexDirection: "column", gap: 6 }}>
{(["pilot", "ekb_full", "manual_list"] as ScopeOption[]).map(
(s) => (
<label
key={s}
style={{
display: "flex",
alignItems: "center",
gap: 8,
fontSize: 14,
cursor: "pointer",
}}
>
<input
type="radio"
name="scope"
value={s}
checked={scope === s}
onChange={() => setScope(s)}
/>
{SCOPE_LABEL[s]}
</label>
),
)}
</div>
</div>
{/* Manual quarters textarea */}
{scope === "manual_list" && (
<label>
<span style={labelStyle}>
Кварталы (по одному на строку или через запятую)
</span>
<textarea
value={quartersText}
onChange={(e) => setQuartersText(e.target.value)}
placeholder={"66:41:0303161\n66:41:0204016"}
rows={6}
style={{
...inputStyle,
fontFamily: "monospace",
fontSize: 12,
}}
/>
</label>
)}
{/* Optional limit */}
<label style={{ maxWidth: 200 }}>
<span style={labelStyle}>
Limit (опц., для pilot default 50)
</span>
<input
type="number"
min={1}
value={limit}
onChange={(e) => setLimit(e.target.value)}
placeholder="50"
style={inputStyle}
/>
</label>
<div style={{ display: "flex", gap: 12, alignItems: "center" }}>
<button
type="submit"
disabled={createJob.isPending || !token}
style={primaryBtn}
>
{createJob.isPending ? "Создание…" : "Запустить"}
</button>
{lastResp && (
<span style={{ fontSize: 13, color: "#5b6066" }}>
Job <code>#{lastResp.job_id}</code> создан, targets:{" "}
<strong>
{lastResp.targets_total.toLocaleString("ru-RU")}
</strong>
, примерно <strong>{lastResp.estimate_minutes}</strong> мин
</span>
)}
{createJob.isError && (
<span style={{ color: "#b3261e", fontSize: 13 }}>
{(createJob.error as Error).message}
</span>
)}
</div>
</div>
</form>
</section>
{/* Jobs table */}
<section style={{ ...cardStyle, marginTop: 16 }}>
<h3 style={sectionTitle}>Cadastre jobs (top 30, refresh 5 с)</h3>
{!token ? (
<p style={{ color: "#5b6066", fontSize: 13 }}>
Введите Admin Token чтобы видеть список jobs.
</p>
) : jobs.isLoading ? (
<p style={{ color: "#5b6066" }}>Загрузка</p>
) : jobs.isError ? (
<p style={{ color: "#b3261e" }}>
Ошибка: {(jobs.error as Error).message}
</p>
) : (
<div style={{ overflowX: "auto" }}>
<table style={tableStyle}>
<thead style={{ background: "#f9fafb" }}>
<tr>
<th style={th}>ID</th>
<th style={th}>Имя</th>
<th style={th}>Scope</th>
<th style={th}>Статус</th>
<th style={{ ...th, textAlign: "right" }}>Progress</th>
<th style={{ ...th, textAlign: "right" }}>Targets</th>
<th style={{ ...th, textAlign: "right" }}>Errors</th>
<th style={{ ...th, textAlign: "right" }}>WAF</th>
<th style={th}>Started</th>
<th style={th}>Heartbeat</th>
<th style={th}></th>
</tr>
</thead>
<tbody>
{jobs.data?.map((job) => (
<>
<tr
key={job.job_id}
style={{
borderTop: "1px solid #f3f4f6",
cursor: "pointer",
background:
expandedId === job.job_id ? "#f0f7ff" : undefined,
}}
onClick={() => toggleExpand(job.job_id)}
>
<td style={{ ...td, fontFamily: "monospace" }}>
{job.job_id}
</td>
<td
style={{
...td,
maxWidth: 180,
overflow: "hidden",
textOverflow: "ellipsis",
whiteSpace: "nowrap",
}}
>
{job.name ?? <em style={{ color: "#9ca3af" }}></em>}
</td>
<td style={{ ...td, fontSize: 11 }}>{job.scope}</td>
<td style={td}>{renderStatusBadge(job)}</td>
<td style={{ ...td, textAlign: "right" }}>
{renderProgressBar(job)}
</td>
<td
style={{
...td,
textAlign: "right",
fontFamily: "monospace",
fontSize: 11,
}}
>
{job.targets_done.toLocaleString("ru-RU")}/
{job.targets_total.toLocaleString("ru-RU")}
{job.targets_failed > 0 && (
<span style={{ color: "#b3261e" }}>
{" "}
({job.targets_failed} fail)
</span>
)}
</td>
<td
style={{
...td,
textAlign: "right",
fontFamily: "monospace",
color: job.targets_failed > 0 ? "#b3261e" : "inherit",
}}
>
{job.targets_failed || "—"}
</td>
<td
style={{
...td,
textAlign: "right",
fontFamily: "monospace",
color:
job.waf_blocked_count > 5
? "#dc2626"
: job.waf_blocked_count > 0
? "#a16207"
: "inherit",
fontWeight: job.waf_blocked_count > 5 ? 700 : 400,
}}
>
{job.waf_blocked_count || "—"}
{job.waf_blocked_count > 5 && " ⚠"}
</td>
<td style={{ ...td, fontSize: 11 }}>
{formatIso(job.started_at)}
</td>
<td style={td}>{renderHeartbeat(job)}</td>
<td style={td} onClick={(e) => e.stopPropagation()}>
{renderActions(job)}
</td>
</tr>
{expandedId === job.job_id &&
renderExpandedPanel(
expandedJob.data?.job_id === job.job_id
? expandedJob.data
: undefined,
)}
</>
))}
{(jobs.data?.length ?? 0) === 0 && (
<tr>
<td
colSpan={11}
style={{
...td,
textAlign: "center",
color: "#6b7280",
padding: 24,
}}
>
Cadastre-jobs ещё не было
</td>
</tr>
)}
</tbody>
</table>
</div>
)}
</section>
</>
);
}
// ---------------------------------------------------------------------------
// Local styles
// ---------------------------------------------------------------------------
const sectionTitle: React.CSSProperties = {
margin: "0 0 12px",
fontSize: 16,
fontWeight: 600,
};
const primaryBtn: React.CSSProperties = {
padding: "10px 18px",
background: "#1d4ed8",
color: "#fff",
border: "none",
borderRadius: 6,
fontSize: 14,
fontWeight: 600,
cursor: "pointer",
};
const secondaryBtn: React.CSSProperties = {
padding: "8px 14px",
background: "#f3f4f6",
color: "#1f2937",
border: "1px solid #d1d5db",
borderRadius: 6,
fontSize: 13,
cursor: "pointer",
};
const tableStyle: React.CSSProperties = {
width: "100%",
borderCollapse: "collapse",
fontSize: 12,
};