gendesign/frontend/src/app/admin/scrape/cadastre/page.tsx
Light1YT 86e9ea2937 fix(week-review): автофиксы код-ревью — 169 issue (label «week ревью 1»)
Многоагентный аудит + имплементация: один воркер на файл, точечные правки.
Верификация: py_compile (47/47 .py) + tsc --noEmit (0 ошибок). Unit-тесты
не прогонялись (окружение не поднято: rollup native dep / нет pytest-venv).

Полностью исправлено (169): #1336, #1337, #1339, #1340, #1341, #1342, #1343, #1345, #1346, #1348, #1349, #1350, #1351, #1354, #1356, #1358, #1359, #1360, #1362, #1364, #1365, #1366, #1367, #1368, #1369, #1370, #1371, #1372, #1373, #1374, #1375, #1376, #1377, #1378, #1379, #1380, #1381, #1382, #1384, #1385, #1386, #1387, #1388, #1389, #1390, #1391, #1392, #1394, #1395, #1396, #1397, #1399, #1400, #1401, #1402, #1403, #1404, #1408, #1409, #1410, #1411, #1412, #1413, #1414, #1415, #1416, #1417, #1418, #1420, #1423, #1425, #1426, #1427, #1428, #1429, #1430, #1431, #1432, #1433, #1434, #1435, #1437, #1438, #1439, #1440, #1441, #1442, #1443, #1444, #1445, #1446, #1447, #1448, #1449, #1450, #1451, #1452, #1453, #1454, #1455, #1456, #1457, #1458, #1459, #1460, #1461, #1462, #1463, #1464, #1465, #1466, #1467, #1468, #1469, #1471, #1472, #1473, #1474, #1476, #1478, #1479, #1481, #1482, #1483, #1484, #1485, #1487, #1488, #1489, #1490, #1491, #1492, #1493, #1494, #1495, #1496, #1497, #1499, #1500, #1501, #1502, #1504, #1505, #1506, #1507, #1510, #1514, #1515, #1516, #1517, #1518, #1519, #1521, #1522, #1523, #1524, #1525, #1526, #1527, #1528, #1529, #1531, #1532, #1533, #1534, #1535, #1536, #1537, #1538

Частично (9, in-file часть, остаток cross-file): #1361, #1419, #1422, #1424, #1470, #1475, #1477, #1480, #1498
Требуют cross-file (3, не тронуты): #1338, #1363, #1421
Пропущено (1): #1539

Не входило в партию: 22 needs-Leha issue (нужны решения владельца).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-15 20:21:11 +05:00

922 lines
29 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"use client";
import { Fragment, useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { apiFetch } from "@/lib/api";
import { cardStyle, inputStyle, labelStyle, td, th } from "@/lib/adminStyles";
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
interface PhaseState {
phase?: string;
parcels_meta?: number;
parcels_fetched?: number;
buildings_meta?: number;
buildings_fetched?: number;
retries?: number;
}
interface CadastreJob {
job_id: number;
name: string | null;
job_kind: string;
scope: string;
status:
| "queued"
| "running"
| "done"
| "failed"
| "zombie"
| "cancelled"
| "paused";
triggered_by: string | null;
rate_ms: number;
created_at: string | null;
started_at: string | null;
finished_at: string | null;
heartbeat_at: string | null;
targets_total: number;
targets_done: number;
targets_failed: number;
targets_skipped: number;
requests_count: number;
waf_blocked_count: number;
error: string | null;
phase_state: PhaseState | null;
progress_pct?: number;
}
interface CreateCadastreJobRequest {
scope: "pilot" | "ekb_full" | "manual_list";
name?: string;
quarters?: string[];
limit?: number;
}
interface CreateCadastreJobResponse {
job_id: number;
scope: string;
targets_total: number;
estimate_minutes: number;
}
type ScopeOption = "pilot" | "ekb_full" | "manual_list";
// ---------------------------------------------------------------------------
// Constants
// ---------------------------------------------------------------------------
const STATUS_COLOR: Record<string, string> = {
queued: "#6b7280",
running: "#1d4ed8",
done: "#16a34a",
failed: "#b3261e",
zombie: "#dc2626",
cancelled: "#9333ea",
paused: "#a16207",
};
const SCOPE_LABEL: Record<ScopeOption, string> = {
pilot: "Pilot (50 кварталов ЕКБ)",
ekb_full: "Full ЕКБ (2 408 кварталов)",
manual_list: "Manual list",
};
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
function formatIso(s: string | null): string {
if (!s) return "—";
return new Date(s).toLocaleString("ru-RU");
}
function diffSecs(a: string | null, b: string | null): string {
if (!a || !b) return "—";
const dt = (new Date(b).getTime() - new Date(a).getTime()) / 1000;
if (dt < 60) return `${dt.toFixed(0)} с`;
if (dt < 3600) return `${(dt / 60).toFixed(1)} мин`;
return `${(dt / 3600).toFixed(1)} ч`;
}
function calcProgress(job: CadastreJob): number {
if (job.progress_pct !== undefined) return job.progress_pct;
if (job.targets_total === 0) return 0;
return (job.targets_done / job.targets_total) * 100;
}
function reqPerSec(job: CadastreJob): string {
if (!job.started_at || job.requests_count === 0) return "—";
const endTs = job.heartbeat_at ?? job.finished_at;
if (!endTs) return "—";
const elapsed =
(new Date(endTs).getTime() - new Date(job.started_at).getTime()) / 1000;
if (elapsed <= 0) return "—";
return (job.requests_count / elapsed).toFixed(2);
}
// ---------------------------------------------------------------------------
// Query keys
// ---------------------------------------------------------------------------
const jobsKey = () => ["admin", "cadastre", "jobs"];
const jobKey = (id: number) => ["admin", "cadastre", "job", id];
// ---------------------------------------------------------------------------
// Component
// ---------------------------------------------------------------------------
export default function CadastreScrapeAdminPage() {
const [scope, setScope] = useState<ScopeOption>("pilot");
const [name, setName] = useState("");
const [quartersText, setQuartersText] = useState("");
const [limit, setLimit] = useState<string>("");
const [lastResp, setLastResp] = useState<CreateCadastreJobResponse | null>(
null,
);
const [expandedId, setExpandedId] = useState<number | null>(null);
const queryClient = useQueryClient();
// -------------------------------------------------------------------------
// Queries
// -------------------------------------------------------------------------
const jobs = useQuery({
queryKey: jobsKey(),
queryFn: () =>
apiFetch<CadastreJob[]>("/api/v1/admin/cadastre/jobs?limit=30"),
refetchInterval: 5000,
});
// Per-job detail (only fetched when row is expanded)
const expandedJob = useQuery({
queryKey: jobKey(expandedId ?? 0),
queryFn: () =>
apiFetch<CadastreJob>(`/api/v1/admin/cadastre/jobs/${expandedId}`),
enabled: expandedId !== null,
refetchInterval: expandedId !== null ? 5000 : false,
});
// -------------------------------------------------------------------------
// Mutations
// -------------------------------------------------------------------------
const createJob = useMutation({
mutationFn: () => {
const body: CreateCadastreJobRequest = { scope };
if (name.trim()) body.name = name.trim();
if (scope === "manual_list") {
body.quarters = quartersText
.split(/[\n,]+/)
.map((s) => s.trim())
.filter(Boolean);
}
const limitNum = parseInt(limit, 10);
if (!isNaN(limitNum) && limitNum > 0) body.limit = limitNum;
return apiFetch<CreateCadastreJobResponse>(
"/api/v1/admin/cadastre/jobs",
{
method: "POST",
body: JSON.stringify(body),
},
);
},
onSuccess: (data) => {
setLastResp(data);
void queryClient.invalidateQueries({ queryKey: jobsKey() });
},
});
const cancelJob = useMutation({
mutationFn: (id: number) =>
apiFetch<{ cancelled: boolean }>(
`/api/v1/admin/cadastre/jobs/${id}/cancel`,
{ method: "POST" },
),
onSuccess: (_data, id) => {
void queryClient.invalidateQueries({ queryKey: jobKey(id) });
void queryClient.invalidateQueries({ queryKey: jobsKey() });
},
});
const resumeJob = useMutation({
mutationFn: (id: number) =>
apiFetch<{ resumed: boolean }>(
`/api/v1/admin/cadastre/jobs/${id}/resume`,
{ method: "POST" },
),
onSuccess: (_data, id) => {
void queryClient.invalidateQueries({ queryKey: jobKey(id) });
void queryClient.invalidateQueries({ queryKey: jobsKey() });
},
});
// -------------------------------------------------------------------------
// Handlers
// -------------------------------------------------------------------------
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault();
if (scope === "manual_list" && !quartersText.trim()) {
alert("Для Manual list укажите хотя бы один квартал");
return;
}
createJob.mutate();
};
const toggleExpand = (id: number) => {
setExpandedId((prev) => (prev === id ? null : id));
};
// -------------------------------------------------------------------------
// Render helpers
// -------------------------------------------------------------------------
const renderProgressBar = (job: CadastreJob) => {
const pct = calcProgress(job);
const color = STATUS_COLOR[job.status] ?? "#6b7280";
return (
<div
style={{
display: "flex",
flexDirection: "column",
gap: 2,
alignItems: "flex-end",
}}
>
<strong>{pct.toFixed(1)}%</strong>
<div
style={{
width: 80,
height: 6,
background: "#e5e7eb",
borderRadius: 3,
overflow: "hidden",
}}
>
<div
style={{
width: `${Math.min(pct, 100)}%`,
height: "100%",
background: color,
}}
/>
</div>
</div>
);
};
const renderHeartbeat = (job: CadastreJob) => {
if (!job.heartbeat_at) return <span style={{ color: "#9ca3af" }}></span>;
const lagMs = Date.now() - new Date(job.heartbeat_at).getTime();
const lagMin = lagMs / 60000;
const stale = job.status === "running" && lagMin > 10;
const lagStr =
lagMs < 60000
? `${Math.round(lagMs / 1000)} с`
: `${lagMin.toFixed(1)} мин`;
return (
<span style={{ color: stale ? "#dc2626" : "inherit", fontSize: 11 }}>
{stale ? `STALE (${lagStr})` : lagStr}
</span>
);
};
const renderStatusBadge = (job: CadastreJob) => {
const color = STATUS_COLOR[job.status] ?? "#6b7280";
const heartbeatLag = job.heartbeat_at
? (Date.now() - new Date(job.heartbeat_at).getTime()) / 60000
: null;
const isZombie =
job.status === "running" && heartbeatLag !== null && heartbeatLag > 10;
return (
<span
style={{
display: "inline-block",
padding: "2px 8px",
borderRadius: 4,
background: color + "22",
color,
fontWeight: 500,
}}
>
{isZombie ? "zombie" : job.status}
</span>
);
};
const renderActions = (job: CadastreJob) => (
<div style={{ display: "flex", gap: 4 }}>
{(job.status === "running" || job.status === "queued") && (
<button
type="button"
onClick={() => cancelJob.mutate(job.job_id)}
disabled={cancelJob.isPending}
style={{ ...secondaryBtn, padding: "4px 8px", fontSize: 11 }}
>
Cancel
</button>
)}
{(job.status === "failed" ||
job.status === "cancelled" ||
job.status === "paused") &&
job.phase_state?.phase !== "done" && (
<button
type="button"
onClick={() => resumeJob.mutate(job.job_id)}
disabled={resumeJob.isPending}
style={{
...secondaryBtn,
padding: "4px 8px",
fontSize: 11,
background: "#dbeafe",
color: "#1d4ed8",
}}
>
Resume
</button>
)}
</div>
);
const renderExpandedPanel = (job: CadastreJob | undefined) => {
if (!job) {
return (
<tr>
<td
colSpan={10}
style={{
...td,
background: "#f9fafb",
fontSize: 12,
color: "#5b6066",
}}
>
Загрузка деталей...
</td>
</tr>
);
}
return (
<tr>
<td
colSpan={10}
style={{
...td,
background: "#f9fafb",
borderBottom: "2px solid #e6e8ec",
}}
>
<div
style={{
display: "grid",
gridTemplateColumns: "repeat(auto-fit, minmax(220px, 1fr))",
gap: 16,
fontSize: 13,
}}
>
{/* Phase state */}
<div>
<div
style={{
fontWeight: 600,
marginBottom: 6,
fontSize: 12,
color: "#5b6066",
}}
>
PHASE STATE
</div>
{job.phase_state ? (
<pre
style={{
margin: 0,
fontFamily: "monospace",
fontSize: 11,
background: "#1e293b",
color: "#e2e8f0",
padding: "8px 10px",
borderRadius: 6,
overflowX: "auto",
}}
>
{JSON.stringify(job.phase_state, null, 2)}
</pre>
) : (
<span style={{ color: "#9ca3af", fontSize: 12 }}></span>
)}
</div>
{/* Stats */}
<div>
<div
style={{
fontWeight: 600,
marginBottom: 6,
fontSize: 12,
color: "#5b6066",
}}
>
СТАТИСТИКА
</div>
<table style={{ fontSize: 12, borderCollapse: "collapse" }}>
<tbody>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Запросов
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
}}
>
{job.requests_count.toLocaleString("ru-RU")}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Req/s
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
}}
>
{reqPerSec(job)}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
WAF blocked
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
color:
job.waf_blocked_count > 5 ? "#dc2626" : "inherit",
fontWeight: job.waf_blocked_count > 5 ? 700 : 400,
}}
>
{job.waf_blocked_count}
{job.waf_blocked_count > 5 && " ⚠ HIGH"}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Skipped
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
}}
>
{job.targets_skipped}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Rate ms
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
}}
>
{job.rate_ms}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Started
</td>
<td style={{ ...td, padding: "3px 0", fontSize: 11 }}>
{formatIso(job.started_at)}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Finished
</td>
<td style={{ ...td, padding: "3px 0", fontSize: 11 }}>
{formatIso(job.finished_at)}
</td>
</tr>
<tr>
<td
style={{
...td,
padding: "3px 12px 3px 0",
color: "#5b6066",
}}
>
Triggered by
</td>
<td
style={{
...td,
padding: "3px 0",
fontFamily: "monospace",
}}
>
{job.triggered_by ?? "—"}
</td>
</tr>
</tbody>
</table>
</div>
{/* Error */}
{job.error && (
<div>
<div
style={{
fontWeight: 600,
marginBottom: 6,
fontSize: 12,
color: "#dc2626",
}}
>
ERROR
</div>
<pre
style={{
margin: 0,
fontFamily: "monospace",
fontSize: 11,
background: "#fef2f2",
color: "#991b1b",
padding: "8px 10px",
borderRadius: 6,
overflowX: "auto",
whiteSpace: "pre-wrap",
wordBreak: "break-all",
}}
>
{job.error}
</pre>
</div>
)}
</div>
</td>
</tr>
);
};
// -------------------------------------------------------------------------
// Render
// -------------------------------------------------------------------------
return (
<>
<h2 style={{ marginTop: 0, fontSize: 18 }}>
Cadastre bulk harvest ручной запуск
</h2>
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 4 }}>
Bulk-загрузка кадастровых данных (участки + здания) из NSPD. State
хранится в БД (<code>cadastre_jobs</code> +{" "}
<code>cadastre_targets</code>) при redeploy job продолжается с
незавершённых targets.
</p>
{/* Create job form */}
<section style={cardStyle}>
<h3 style={sectionTitle}>Запуск bulk harvest</h3>
<form onSubmit={handleSubmit}>
<div style={{ display: "grid", gap: 12 }}>
<label>
<span style={labelStyle}>Имя задачи (опционально)</span>
<input
type="text"
value={name}
onChange={(e) => setName(e.target.value)}
placeholder={`pilot_${new Date().toISOString().slice(0, 10)}`}
style={inputStyle}
/>
</label>
{/* Scope radio buttons */}
<div>
<span style={labelStyle}>Scope</span>
<div style={{ display: "flex", flexDirection: "column", gap: 6 }}>
{(["pilot", "ekb_full", "manual_list"] as ScopeOption[]).map(
(s) => (
<label
key={s}
style={{
display: "flex",
alignItems: "center",
gap: 8,
fontSize: 14,
cursor: "pointer",
}}
>
<input
type="radio"
name="scope"
value={s}
checked={scope === s}
onChange={() => setScope(s)}
/>
{SCOPE_LABEL[s]}
</label>
),
)}
</div>
</div>
{/* Manual quarters textarea */}
{scope === "manual_list" && (
<label>
<span style={labelStyle}>
Кварталы (по одному на строку или через запятую)
</span>
<textarea
value={quartersText}
onChange={(e) => setQuartersText(e.target.value)}
placeholder={"66:41:0303161\n66:41:0204016"}
rows={6}
style={{
...inputStyle,
fontFamily: "monospace",
fontSize: 12,
}}
/>
</label>
)}
{/* Optional limit */}
<label style={{ maxWidth: 200 }}>
<span style={labelStyle}>
Limit (опц., для pilot default 50)
</span>
<input
type="number"
min={1}
value={limit}
onChange={(e) => setLimit(e.target.value)}
placeholder="50"
style={inputStyle}
/>
</label>
<div style={{ display: "flex", gap: 12, alignItems: "center" }}>
<button
type="submit"
disabled={createJob.isPending}
style={primaryBtn}
>
{createJob.isPending ? "Создание…" : "Запустить"}
</button>
{lastResp && (
<span style={{ fontSize: 13, color: "#5b6066" }}>
Job <code>#{lastResp.job_id}</code> создан, targets:{" "}
<strong>
{lastResp.targets_total.toLocaleString("ru-RU")}
</strong>
, примерно <strong>{lastResp.estimate_minutes}</strong> мин
</span>
)}
{createJob.isError && (
<span style={{ color: "#b3261e", fontSize: 13 }}>
{(createJob.error as Error).message}
</span>
)}
</div>
</div>
</form>
</section>
{/* Jobs table */}
<section style={{ ...cardStyle, marginTop: 16 }}>
<h3 style={sectionTitle}>Cadastre jobs (top 30, refresh 5 с)</h3>
{jobs.isLoading ? (
<p style={{ color: "#5b6066" }}>Загрузка</p>
) : jobs.isError ? (
<p style={{ color: "#b3261e" }}>
Ошибка: {(jobs.error as Error).message}
</p>
) : (
<div style={{ overflowX: "auto" }}>
<table style={tableStyle}>
<thead style={{ background: "#f9fafb" }}>
<tr>
<th style={th}>ID</th>
<th style={th}>Имя</th>
<th style={th}>Scope</th>
<th style={th}>Статус</th>
<th style={{ ...th, textAlign: "right" }}>Progress</th>
<th style={{ ...th, textAlign: "right" }}>Targets</th>
<th style={{ ...th, textAlign: "right" }}>Errors</th>
<th style={{ ...th, textAlign: "right" }}>WAF</th>
<th style={th}>Started</th>
<th style={th}>Heartbeat</th>
<th style={th}></th>
</tr>
</thead>
<tbody>
{jobs.data?.map((job) => (
<Fragment key={job.job_id}>
<tr
style={{
borderTop: "1px solid #f3f4f6",
cursor: "pointer",
background:
expandedId === job.job_id ? "#f0f7ff" : undefined,
}}
onClick={() => toggleExpand(job.job_id)}
>
<td style={{ ...td, fontFamily: "monospace" }}>
{job.job_id}
</td>
<td
style={{
...td,
maxWidth: 180,
overflow: "hidden",
textOverflow: "ellipsis",
whiteSpace: "nowrap",
}}
>
{job.name ?? <em style={{ color: "#9ca3af" }}></em>}
</td>
<td style={{ ...td, fontSize: 11 }}>{job.scope}</td>
<td style={td}>{renderStatusBadge(job)}</td>
<td style={{ ...td, textAlign: "right" }}>
{renderProgressBar(job)}
</td>
<td
style={{
...td,
textAlign: "right",
fontFamily: "monospace",
fontSize: 11,
}}
>
{job.targets_done.toLocaleString("ru-RU")}/
{job.targets_total.toLocaleString("ru-RU")}
{job.targets_failed > 0 && (
<span style={{ color: "#b3261e" }}>
{" "}
({job.targets_failed} fail)
</span>
)}
</td>
<td
style={{
...td,
textAlign: "right",
fontFamily: "monospace",
color: job.targets_failed > 0 ? "#b3261e" : "inherit",
}}
>
{job.targets_failed || "—"}
</td>
<td
style={{
...td,
textAlign: "right",
fontFamily: "monospace",
color:
job.waf_blocked_count > 5
? "#dc2626"
: job.waf_blocked_count > 0
? "#a16207"
: "inherit",
fontWeight: job.waf_blocked_count > 5 ? 700 : 400,
}}
>
{job.waf_blocked_count || "—"}
{job.waf_blocked_count > 5 && " ⚠"}
</td>
<td style={{ ...td, fontSize: 11 }}>
{formatIso(job.started_at)}
</td>
<td style={td}>{renderHeartbeat(job)}</td>
<td style={td} onClick={(e) => e.stopPropagation()}>
{renderActions(job)}
</td>
</tr>
{expandedId === job.job_id &&
renderExpandedPanel(
expandedJob.data?.job_id === job.job_id
? expandedJob.data
: undefined,
)}
</Fragment>
))}
{(jobs.data?.length ?? 0) === 0 && (
<tr>
<td
colSpan={11}
style={{
...td,
textAlign: "center",
color: "#6b7280",
padding: 24,
}}
>
Cadastre-jobs ещё не было
</td>
</tr>
)}
</tbody>
</table>
</div>
)}
</section>
</>
);
}
// ---------------------------------------------------------------------------
// Local styles
// ---------------------------------------------------------------------------
const sectionTitle: React.CSSProperties = {
margin: "0 0 12px",
fontSize: 16,
fontWeight: 600,
};
const primaryBtn: React.CSSProperties = {
padding: "10px 18px",
background: "#1d4ed8",
color: "#fff",
border: "none",
borderRadius: 6,
fontSize: 14,
fontWeight: 600,
cursor: "pointer",
};
const secondaryBtn: React.CSSProperties = {
padding: "8px 14px",
background: "#f3f4f6",
color: "#1f2937",
border: "1px solid #d1d5db",
borderRadius: 6,
fontSize: 13,
cursor: "pointer",
};
const tableStyle: React.CSSProperties = {
width: "100%",
borderCollapse: "collapse",
fontSize: 12,
};