79e2e94e09
УДАЛЕНО (~1700 строк): * backend/app/services/scrapers/nspd_kn.py (Playwright + WAF-bypass) * backend/app/workers/tasks/scrape_nspd.py * frontend/src/app/admin/scrape/nspd/page.tsx * 5 NSPD admin endpoints (POST /nspd, /nspd/release-lock, GET /nspd/runs, /logs, /coverage) * NSPD beat schedule + worker_ready resume hook + nav tab + link * _nspd_default_regions() helper СОХРАНЕНО (история): * nspd_scrape_runs / nspd_scrape_log таблицы (3 row + log) для аудита * UNION ALL в v_scrape_runs_unified / v_scrape_log_unified — старые runs видны под фильтром scraper_type='nspd' * settings scrape_nspd_* помечены DEPRECATED CHANGED: * nspd_geo_jobs.use_rosreestr2coord DEFAULT FALSE → TRUE * UI checkbox "rosreestr2coord lib" по умолчанию = checked * /admin/scrape/all + /admin/scrape/geo — единственный путь к NSPD-данным NSPD-доступ теперь только через rosreestr2coord (community lib) — upstream поддерживает обновления headers/WAF-tricks. Локальный nspd_lite.py (urllib) остаётся как fallback (use_rosreestr2coord=FALSE).
544 lines
17 KiB
TypeScript
544 lines
17 KiB
TypeScript
"use client";
|
||
|
||
import { useState } from "react";
|
||
import { useQuery } from "@tanstack/react-query";
|
||
|
||
import { apiFetch } from "@/lib/api";
|
||
|
||
interface UnifiedRunRow {
|
||
scraper_type: "kn" | "nspd" | "objective" | "nspd_geo";
|
||
run_id: number;
|
||
started_at: string | null;
|
||
finished_at: string | null;
|
||
heartbeat_at: string | null;
|
||
status: string;
|
||
error: string | null;
|
||
triggered_by: string | null;
|
||
scope: string | null;
|
||
requests_count: number | null;
|
||
items_ok: number | null;
|
||
items_failed: number | null;
|
||
sub_items_ok: number | null;
|
||
progress_idx: number | null;
|
||
progress_total: number | null;
|
||
extra: Record<string, unknown> | null;
|
||
}
|
||
|
||
interface UnifiedLogRow {
|
||
scraper_type: "kn" | "nspd" | "nspd_geo";
|
||
log_id: number;
|
||
run_id: number | null;
|
||
ts: string | null;
|
||
level: "debug" | "info" | "warn" | "error";
|
||
stage: string | null;
|
||
entity_id: string | null;
|
||
message: string;
|
||
}
|
||
|
||
const SCRAPER_BADGE: Record<string, { bg: string; fg: string; label: string }> =
|
||
{
|
||
kn: { bg: "#dbeafe", fg: "#1d4ed8", label: "DOM.РФ kn" },
|
||
nspd: { bg: "#fef3c7", fg: "#a16207", label: "NSPD" },
|
||
objective: { bg: "#d1fae5", fg: "#059669", label: "Objective" },
|
||
nspd_geo: { bg: "#ede9fe", fg: "#6d28d9", label: "🗺 NSPD geo" },
|
||
};
|
||
|
||
const STATUS_COLOR: Record<string, string> = {
|
||
done: "#16a34a",
|
||
running: "#1d4ed8",
|
||
failed: "#b3261e",
|
||
zombie: "#9333ea",
|
||
};
|
||
|
||
function formatIso(s: string | null): string {
|
||
if (!s) return "—";
|
||
return new Date(s).toLocaleString("ru-RU");
|
||
}
|
||
|
||
function diffSecs(a: string | null, b: string | null): string {
|
||
if (!a || !b) return "—";
|
||
const dt = (new Date(b).getTime() - new Date(a).getTime()) / 1000;
|
||
if (dt < 60) return `${dt.toFixed(0)}с`;
|
||
if (dt < 3600) return `${(dt / 60).toFixed(1)} мин`;
|
||
return `${(dt / 3600).toFixed(1)} ч`;
|
||
}
|
||
|
||
export default function ScrapeAllAdminPage() {
|
||
const [token, setToken] = useState<string>(() =>
|
||
typeof window === "undefined"
|
||
? ""
|
||
: (localStorage.getItem("admin_token") ?? ""),
|
||
);
|
||
const [filter, setFilter] = useState<
|
||
"all" | "kn" | "nspd" | "objective" | "nspd_geo"
|
||
>("all");
|
||
const [logsRunId, setLogsRunId] = useState<number | "">("");
|
||
|
||
const saveToken = (v: string) => {
|
||
setToken(v);
|
||
localStorage.setItem("admin_token", v);
|
||
};
|
||
|
||
const runs = useQuery({
|
||
queryKey: ["unified-runs", token, filter],
|
||
queryFn: () => {
|
||
const q =
|
||
filter !== "all" ? `?scraper_type=${filter}&limit=30` : "?limit=30";
|
||
return apiFetch<UnifiedRunRow[]>(`/api/v1/admin/scrape/all/runs${q}`, {
|
||
headers: { "X-Admin-Token": token },
|
||
});
|
||
},
|
||
enabled: !!token,
|
||
refetchInterval: 5000,
|
||
});
|
||
|
||
const logs = useQuery({
|
||
queryKey: ["unified-logs", token, filter, logsRunId],
|
||
queryFn: () => {
|
||
const params = new URLSearchParams({ limit: "100" });
|
||
if (filter !== "all") params.set("scraper_type", filter);
|
||
if (logsRunId !== "") params.set("run_id", String(logsRunId));
|
||
return apiFetch<UnifiedLogRow[]>(
|
||
`/api/v1/admin/scrape/all/logs?${params}`,
|
||
{ headers: { "X-Admin-Token": token } },
|
||
);
|
||
},
|
||
enabled: !!token,
|
||
refetchInterval: 8000,
|
||
});
|
||
|
||
// Сводка по statusам
|
||
const summary = (() => {
|
||
const s = { running: 0, done: 0, failed: 0, zombie: 0 } as Record<
|
||
string,
|
||
number
|
||
>;
|
||
runs.data?.forEach((r) => {
|
||
s[r.status] = (s[r.status] ?? 0) + 1;
|
||
});
|
||
return s;
|
||
})();
|
||
|
||
return (
|
||
<>
|
||
<h2 style={{ marginTop: 0, fontSize: 18 }}>
|
||
Скраперы — сводный dashboard
|
||
</h2>
|
||
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 4 }}>
|
||
Все скраперы в одной таблице (через <code>v_scrape_runs_unified</code>).
|
||
Для запуска / триггеров — отдельные страницы:{" "}
|
||
<a href="/admin/scrape" style={{ color: "#1d4ed8" }}>
|
||
DOM.РФ kn
|
||
</a>{" "}
|
||
·{" "}
|
||
<a href="/admin/scrape/geo" style={{ color: "#1d4ed8" }}>
|
||
🗺 NSPD geo
|
||
</a>{" "}
|
||
·{" "}
|
||
<a href="/admin/scrape/objective" style={{ color: "#1d4ed8" }}>
|
||
Objective
|
||
</a>
|
||
. Старый Playwright-based NSPD-скрапер снят с эксплуатации 2026-05-11 —
|
||
история его прогонов осталась под фильтром <code>nspd</code> только для
|
||
аудита.
|
||
</p>
|
||
|
||
<section style={cardStyle}>
|
||
<label style={{ display: "block" }}>
|
||
<span style={labelStyle}>Admin Token</span>
|
||
<input
|
||
type="password"
|
||
value={token}
|
||
onChange={(e) => saveToken(e.target.value)}
|
||
placeholder="SCRAPE_ADMIN_TOKEN"
|
||
style={inputStyle}
|
||
/>
|
||
</label>
|
||
</section>
|
||
|
||
{/* Сводка-плитки */}
|
||
<section
|
||
style={{
|
||
...cardStyle,
|
||
marginTop: 16,
|
||
display: "grid",
|
||
gridTemplateColumns: "repeat(4, 1fr)",
|
||
gap: 16,
|
||
}}
|
||
>
|
||
<Stat
|
||
label="🟦 в работе"
|
||
value={summary.running ?? 0}
|
||
color="#1d4ed8"
|
||
/>
|
||
<Stat label="✅ успешно" value={summary.done ?? 0} color="#16a34a" />
|
||
<Stat label="❌ failed" value={summary.failed ?? 0} color="#b3261e" />
|
||
<Stat label="🧟 zombie" value={summary.zombie ?? 0} color="#9333ea" />
|
||
</section>
|
||
|
||
{/* Фильтр + табы */}
|
||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||
<div style={{ display: "flex", gap: 8, alignItems: "center" }}>
|
||
<span style={{ ...labelStyle, marginBottom: 0 }}>Filter:</span>
|
||
{(["all", "kn", "nspd", "nspd_geo", "objective"] as const).map(
|
||
(t) => (
|
||
<button
|
||
key={t}
|
||
type="button"
|
||
onClick={() => setFilter(t)}
|
||
style={{
|
||
...filterTab,
|
||
background: filter === t ? "#1d4ed8" : "#f3f4f6",
|
||
color: filter === t ? "#fff" : "#1f2937",
|
||
}}
|
||
>
|
||
{t === "all" ? "Все" : (SCRAPER_BADGE[t]?.label ?? t)}
|
||
</button>
|
||
),
|
||
)}
|
||
</div>
|
||
</section>
|
||
|
||
{/* Таблица прогонов */}
|
||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||
<h3 style={sectionTitle}>
|
||
📜 Последние прогоны (top 30, обновляется каждые 5с)
|
||
</h3>
|
||
<div style={{ overflowX: "auto" }}>
|
||
<table style={tableStyle}>
|
||
<thead style={{ background: "#f9fafb" }}>
|
||
<tr>
|
||
<th style={th}>scraper</th>
|
||
<th style={th}>run_id</th>
|
||
<th style={th}>started</th>
|
||
<th style={th}>scope</th>
|
||
<th style={th}>duration</th>
|
||
<th style={{ ...th, textAlign: "right" }}>items_ok</th>
|
||
<th style={{ ...th, textAlign: "right" }}>failed</th>
|
||
<th style={{ ...th, textAlign: "right" }}>sub_ok</th>
|
||
<th style={th}>status</th>
|
||
<th style={th}>trigger</th>
|
||
<th style={th}></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
{runs.data?.map((r) => {
|
||
const heartbeatLag = r.heartbeat_at
|
||
? (Date.now() - new Date(r.heartbeat_at).getTime()) / 1000
|
||
: null;
|
||
const isStale =
|
||
r.status === "running" &&
|
||
heartbeatLag !== null &&
|
||
heartbeatLag > 60;
|
||
const badge = SCRAPER_BADGE[r.scraper_type];
|
||
const statusColor = STATUS_COLOR[r.status] ?? "#6b7280";
|
||
return (
|
||
<tr
|
||
key={`${r.scraper_type}-${r.run_id}`}
|
||
style={{ borderTop: "1px solid #f3f4f6" }}
|
||
>
|
||
<td style={td}>
|
||
{badge && (
|
||
<span
|
||
style={{
|
||
display: "inline-block",
|
||
padding: "2px 8px",
|
||
borderRadius: 4,
|
||
background: badge.bg,
|
||
color: badge.fg,
|
||
fontWeight: 500,
|
||
fontSize: 11,
|
||
}}
|
||
>
|
||
{badge.label}
|
||
</span>
|
||
)}
|
||
</td>
|
||
<td style={{ ...td, fontFamily: "monospace" }}>
|
||
{r.run_id}
|
||
</td>
|
||
<td style={td}>{formatIso(r.started_at)}</td>
|
||
<td
|
||
style={{ ...td, fontFamily: "monospace", fontSize: 11 }}
|
||
>
|
||
{r.scope ?? "—"}
|
||
</td>
|
||
<td style={td}>
|
||
{diffSecs(r.started_at, r.finished_at ?? r.heartbeat_at)}
|
||
</td>
|
||
<td
|
||
style={{
|
||
...td,
|
||
textAlign: "right",
|
||
fontFamily: "monospace",
|
||
}}
|
||
>
|
||
{r.items_ok?.toLocaleString("ru-RU") ?? "—"}
|
||
</td>
|
||
<td
|
||
style={{
|
||
...td,
|
||
textAlign: "right",
|
||
fontFamily: "monospace",
|
||
color: r.items_failed ? "#b3261e" : "inherit",
|
||
}}
|
||
>
|
||
{r.items_failed?.toLocaleString("ru-RU") ?? "—"}
|
||
</td>
|
||
<td
|
||
style={{
|
||
...td,
|
||
textAlign: "right",
|
||
fontFamily: "monospace",
|
||
}}
|
||
>
|
||
{r.sub_items_ok?.toLocaleString("ru-RU") ?? "—"}
|
||
</td>
|
||
<td style={td}>
|
||
<span
|
||
style={{
|
||
display: "inline-block",
|
||
padding: "2px 8px",
|
||
borderRadius: 4,
|
||
background: statusColor + "22",
|
||
color: statusColor,
|
||
fontWeight: 500,
|
||
}}
|
||
>
|
||
{isStale ? "stale" : r.status}
|
||
</span>
|
||
</td>
|
||
<td style={{ ...td, color: "#6b7280" }}>
|
||
{r.triggered_by ?? "—"}
|
||
</td>
|
||
<td style={td}>
|
||
{(r.scraper_type === "kn" ||
|
||
r.scraper_type === "nspd" ||
|
||
r.scraper_type === "nspd_geo") && (
|
||
<button
|
||
type="button"
|
||
onClick={() => setLogsRunId(r.run_id)}
|
||
style={{
|
||
...secondaryBtn,
|
||
padding: "4px 10px",
|
||
fontSize: 11,
|
||
}}
|
||
>
|
||
логи
|
||
</button>
|
||
)}
|
||
</td>
|
||
</tr>
|
||
);
|
||
})}
|
||
{runs.data?.length === 0 && (
|
||
<tr>
|
||
<td
|
||
colSpan={11}
|
||
style={{
|
||
...td,
|
||
textAlign: "center",
|
||
color: "#6b7280",
|
||
padding: 24,
|
||
}}
|
||
>
|
||
Прогонов с фильтром <code>{filter}</code> нет
|
||
</td>
|
||
</tr>
|
||
)}
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
</section>
|
||
|
||
{/* Logs */}
|
||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||
<h3 style={sectionTitle}>
|
||
📋 Логи (top 100){" "}
|
||
{logsRunId !== "" && (
|
||
<span style={{ fontSize: 13, fontWeight: 400, color: "#5b6066" }}>
|
||
· фильтр run_id={logsRunId}{" "}
|
||
<button
|
||
type="button"
|
||
onClick={() => setLogsRunId("")}
|
||
style={{
|
||
background: "none",
|
||
border: "none",
|
||
color: "#1d4ed8",
|
||
cursor: "pointer",
|
||
fontSize: 13,
|
||
padding: 0,
|
||
}}
|
||
>
|
||
сбросить
|
||
</button>
|
||
</span>
|
||
)}
|
||
</h3>
|
||
<div style={{ overflowX: "auto", maxHeight: 400, overflowY: "auto" }}>
|
||
<table style={tableStyle}>
|
||
<thead
|
||
style={{ background: "#f9fafb", position: "sticky", top: 0 }}
|
||
>
|
||
<tr>
|
||
<th style={th}>ts</th>
|
||
<th style={th}>scraper</th>
|
||
<th style={th}>run</th>
|
||
<th style={th}>level</th>
|
||
<th style={th}>stage</th>
|
||
<th style={th}>entity</th>
|
||
<th style={th}>message</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
{logs.data?.map((l) => (
|
||
<tr
|
||
key={`${l.scraper_type}-${l.log_id}`}
|
||
style={{ borderTop: "1px solid #f3f4f6" }}
|
||
>
|
||
<td style={{ ...td, fontSize: 11, whiteSpace: "nowrap" }}>
|
||
{formatIso(l.ts)}
|
||
</td>
|
||
<td style={td}>
|
||
<span
|
||
style={{
|
||
fontSize: 10,
|
||
color: SCRAPER_BADGE[l.scraper_type]?.fg ?? "#6b7280",
|
||
}}
|
||
>
|
||
{l.scraper_type}
|
||
</span>
|
||
</td>
|
||
<td style={{ ...td, fontFamily: "monospace" }}>
|
||
{l.run_id ?? "—"}
|
||
</td>
|
||
<td style={td}>
|
||
<span
|
||
style={{
|
||
color:
|
||
l.level === "error"
|
||
? "#b3261e"
|
||
: l.level === "warn"
|
||
? "#a16207"
|
||
: "#5b6066",
|
||
fontSize: 11,
|
||
fontWeight: 500,
|
||
}}
|
||
>
|
||
{l.level}
|
||
</span>
|
||
</td>
|
||
<td style={{ ...td, fontSize: 11 }}>{l.stage ?? "—"}</td>
|
||
<td style={{ ...td, fontFamily: "monospace", fontSize: 11 }}>
|
||
{l.entity_id ?? "—"}
|
||
</td>
|
||
<td style={{ ...td, fontSize: 12 }}>{l.message}</td>
|
||
</tr>
|
||
))}
|
||
{logs.data?.length === 0 && (
|
||
<tr>
|
||
<td
|
||
colSpan={7}
|
||
style={{
|
||
...td,
|
||
textAlign: "center",
|
||
color: "#6b7280",
|
||
padding: 24,
|
||
}}
|
||
>
|
||
Логов нет (objective пока не пишет в log — только runs)
|
||
</td>
|
||
</tr>
|
||
)}
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
</section>
|
||
</>
|
||
);
|
||
}
|
||
|
||
function Stat({
|
||
label,
|
||
value,
|
||
color,
|
||
}: {
|
||
label: string;
|
||
value: number;
|
||
color: string;
|
||
}) {
|
||
return (
|
||
<div>
|
||
<div style={{ fontSize: 12, color: "#5b6066", marginBottom: 4 }}>
|
||
{label}
|
||
</div>
|
||
<div
|
||
style={{
|
||
fontFamily: "monospace",
|
||
fontSize: 28,
|
||
fontWeight: 600,
|
||
color,
|
||
}}
|
||
>
|
||
{value}
|
||
</div>
|
||
</div>
|
||
);
|
||
}
|
||
|
||
const cardStyle = {
|
||
background: "#fff",
|
||
border: "1px solid #e6e8ec",
|
||
borderRadius: 12,
|
||
padding: 20,
|
||
};
|
||
const sectionTitle = {
|
||
margin: "0 0 12px",
|
||
fontSize: 16,
|
||
fontWeight: 600 as const,
|
||
};
|
||
const labelStyle = {
|
||
display: "block",
|
||
fontSize: 12,
|
||
color: "#5b6066",
|
||
marginBottom: 4,
|
||
textTransform: "uppercase" as const,
|
||
letterSpacing: 0.4,
|
||
};
|
||
const inputStyle = {
|
||
padding: "8px 10px",
|
||
border: "1px solid #d1d5db",
|
||
borderRadius: 6,
|
||
fontSize: 14,
|
||
width: "100%",
|
||
boxSizing: "border-box" as const,
|
||
};
|
||
const filterTab = {
|
||
padding: "6px 14px",
|
||
border: "none",
|
||
borderRadius: 6,
|
||
fontSize: 13,
|
||
fontWeight: 500 as const,
|
||
cursor: "pointer",
|
||
};
|
||
const secondaryBtn = {
|
||
padding: "8px 14px",
|
||
background: "#f3f4f6",
|
||
color: "#1f2937",
|
||
border: "1px solid #d1d5db",
|
||
borderRadius: 6,
|
||
fontSize: 13,
|
||
cursor: "pointer",
|
||
};
|
||
const tableStyle = {
|
||
width: "100%",
|
||
borderCollapse: "collapse" as const,
|
||
fontSize: 12,
|
||
};
|
||
const th = {
|
||
padding: "8px 10px",
|
||
textAlign: "left" as const,
|
||
fontWeight: 600 as const,
|
||
borderBottom: "1px solid #e6e8ec",
|
||
};
|
||
const td = { padding: "8px 10px" };
|