gendesign/frontend/src/app/admin/scrape/objective/page.tsx
lekss361 df34ad7bb5 refactor(security): убрать UI ввода admin token (backend не требует с PR #437)
- Удалить token state + localStorage.setItem("admin_token") из всех admin pages
- Удалить X-Admin-Token header из всех apiFetch вызовов
- Убрать enabled: !!token из useQuery — fix «Загрузка…» forever для новых users
- Убрать token prop из BulkGeoPanel, JobSettingsPanel, ScrapeLogsPanel
- Удалить !token guards и UI input fields для ввода токена
- queryKey: убрать token из ключей (scrape-runs, scrape-queue, etc.)
2026-05-23 14:57:12 +03:00

807 lines
28 KiB
TypeScript
Raw Blame History

This file contains invisible Unicode characters

This file contains invisible Unicode characters that are indistinguishable to humans but may be processed differently by a computer. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"use client";
import { useEffect, useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import ScrapeLogsPanel from "@/components/admin/ScrapeLogsPanel";
import { apiFetch } from "@/lib/api";
import { cardStyle, inputStyle, labelStyle, td, th } from "@/lib/adminStyles";
interface ObjectiveRunRow {
run_id: number;
started_at: string | null;
finished_at: string | null;
heartbeat_at: string | null;
group_name: string;
reports_ok: number;
reports_failed: number;
rows_corpus_room: number;
rows_lots: number;
rows_mappings: number;
requests_count: number;
status: string;
error: string | null;
triggered_by: string;
}
interface ObjectiveCoverage {
pg: {
lots: number;
corp_room_month: number;
mappings: number;
last_lot_snapshot: string | null;
last_run_at: string | null;
};
sqlite: {
path: string;
exists: boolean;
size_bytes?: number;
modified_at?: number;
lots?: number;
corp_room_month?: number;
mappings?: number;
error?: string;
};
}
interface ObjectiveTriggerResp {
task_id: string;
sqlite_path: string;
}
interface ObjectiveSyncOurResp {
task_id: string;
groups_planned: string[];
estimate_minutes: number;
}
interface ObjectiveConfig {
cron_schedule: string;
groups_csv: string;
groups: string[];
use_ddu: boolean;
use_dkp: boolean;
period_months_back: number;
inter_group_delay_s: number;
rate_ms: number;
retries: number;
updated_at: string | null;
updated_by: string | null;
_note?: string;
}
function formatBytes(b: number | undefined): string {
if (!b) return "—";
const mb = b / 1024 / 1024;
return mb >= 100 ? `${mb.toFixed(0)} MB` : `${mb.toFixed(1)} MB`;
}
function formatEpoch(sec: number | undefined): string {
if (!sec) return "—";
return new Date(sec * 1000).toLocaleString("ru-RU");
}
function formatIso(s: string | null): string {
if (!s) return "—";
return new Date(s).toLocaleString("ru-RU");
}
function diffSecs(a: string | null, b: string | null): string {
if (!a || !b) return "—";
const dt = (new Date(b).getTime() - new Date(a).getTime()) / 1000;
if (dt < 60) return `${dt.toFixed(1)}с`;
return `${(dt / 60).toFixed(1)} мин`;
}
export default function ObjectiveEtlAdminPage() {
const [overrideSqlitePath, setOverrideSqlitePath] = useState("");
const [lastTaskId, setLastTaskId] = useState<string | null>(null);
const [syncGroupsCsv, setSyncGroupsCsv] = useState("");
const [lastSyncTaskId, setLastSyncTaskId] = useState<string | null>(null);
const [cfgDraft, setCfgDraft] = useState<Partial<ObjectiveConfig>>({});
const [cfgSaved, setCfgSaved] = useState(false);
const [showEtl, setShowEtl] = useState(false);
const queryClient = useQueryClient();
const coverage = useQuery({
queryKey: ["objective-coverage"],
queryFn: () =>
apiFetch<ObjectiveCoverage>("/api/v1/admin/scrape/objective/coverage"),
refetchInterval: 10000,
});
const runs = useQuery({
queryKey: ["objective-runs"],
queryFn: () =>
apiFetch<ObjectiveRunRow[]>(
"/api/v1/admin/scrape/objective/runs?limit=20",
),
refetchInterval: 5000,
});
const cfg = useQuery({
queryKey: ["objective-config"],
queryFn: () =>
apiFetch<ObjectiveConfig>("/api/v1/admin/scrape/objective/config"),
refetchOnWindowFocus: false,
});
// Подтягиваем cfg в draft при первом появлении (не во время render!)
useEffect(() => {
if (cfg.data && Object.keys(cfgDraft).length === 0) {
setCfgDraft(cfg.data);
}
}, [cfg.data, cfgDraft]);
const trigger = useMutation({
mutationFn: () =>
apiFetch<ObjectiveTriggerResp>("/api/v1/admin/scrape/objective", {
method: "POST",
body: JSON.stringify({ sqlite_path: overrideSqlitePath || null }),
}),
onSuccess: (data) => {
setLastTaskId(data.task_id);
void queryClient.invalidateQueries({ queryKey: ["objective-runs"] });
void queryClient.invalidateQueries({ queryKey: ["objective-coverage"] });
},
});
const syncOur = useMutation({
mutationFn: () => {
const groups = syncGroupsCsv
.split(",")
.map((g) => g.trim())
.filter(Boolean);
return apiFetch<ObjectiveSyncOurResp>(
"/api/v1/admin/scrape/objective/sync-our",
{
method: "POST",
body: JSON.stringify({ groups: groups.length ? groups : null }),
},
);
},
onSuccess: (data) => {
setLastSyncTaskId(data.task_id);
void queryClient.invalidateQueries({ queryKey: ["objective-runs"] });
},
});
const saveCfg = useMutation({
mutationFn: (patch: Partial<ObjectiveConfig>) =>
apiFetch<ObjectiveConfig>("/api/v1/admin/scrape/objective/config", {
method: "PUT",
body: JSON.stringify(patch),
}),
onSuccess: (data) => {
setCfgDraft(data);
setCfgSaved(true);
window.setTimeout(() => setCfgSaved(false), 3000);
void queryClient.invalidateQueries({ queryKey: ["objective-config"] });
},
});
return (
<>
<h2 style={{ marginTop: 0, fontSize: 18 }}>
Objective: sync с api.objctv.ru наша PG
</h2>
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 4 }}>
<strong>Source of truth = эта PG.</strong> Наш Celery каждый вторник
06:00 МСК тянет 4 группы (Свердл.обл + Челябинск + Тюмень + Пермь) с
api.objctv.ru напрямую в objective_lots / corpus_room_month /
lots_history. SQLite Антона (/sf/) отдельный микросервис, остался
только как bootstrap-fallback (свёрнут ниже).
</p>
{/* Coverage */}
<section style={cardStyle}>
<h3 style={sectionTitle}>📦 Состояние PG (источник истины)</h3>
{coverage.isLoading && (
<p style={{ color: "#5b6066", fontSize: 13 }}>Загрузка</p>
)}
{coverage.data && (
<div
style={{
display: "grid",
gridTemplateColumns: "1fr 1fr 1fr",
gap: 24,
}}
>
<Stat label="objective_lots" value={coverage.data.pg.lots} />
<Stat
label="corp×room×month"
value={coverage.data.pg.corp_room_month}
/>
<Stat label="complex_mapping" value={coverage.data.pg.mappings} />
<div
style={{
gridColumn: "1 / -1",
borderTop: "1px solid #e6e8ec",
paddingTop: 12,
fontSize: 12,
color: "#5b6066",
display: "grid",
gridTemplateColumns: "1fr 1fr",
gap: 16,
}}
>
<div>
last lot snapshot:{" "}
<code>{coverage.data.pg.last_lot_snapshot ?? "—"}</code>
</div>
<div>
last successful sync:{" "}
<code>{formatIso(coverage.data.pg.last_run_at)}</code>
</div>
</div>
<div
style={{
gridColumn: "1 / -1",
fontSize: 11,
color: "#9ca3af",
display: "grid",
gridTemplateColumns: "1fr 1fr 1fr",
gap: 16,
}}
>
<div>
SQLite Антона (legacy):{" "}
{coverage.data.sqlite.exists ? (
<span style={{ color: "#16a34a" }}>
{formatBytes(coverage.data.sqlite.size_bytes)}
</span>
) : (
<span style={{ color: "#9ca3af" }}> не примонтирован</span>
)}
</div>
<div>
файл обновлён:{" "}
<code>{formatEpoch(coverage.data.sqlite.modified_at)}</code>
</div>
<div>
lots в SQLite:{" "}
<code>
{coverage.data.sqlite.lots?.toLocaleString("ru-RU") ?? "—"}
</code>
</div>
</div>
</div>
)}
</section>
{/* ⚙️ Settings — основной блок (всегда раскрыт) */}
<section style={{ ...cardStyle, marginTop: 16, borderColor: "#3b82f6" }}>
<h3 style={{ ...sectionTitle, color: "#1d4ed8" }}>
Настройки sync расписание + параметры запроса
</h3>
<p
style={{
color: "#5b6066",
fontSize: 12,
marginTop: -4,
marginBottom: 16,
}}
>
Хранятся в БД <code>objective_sync_config</code>. Все поля кроме cron
подхватываются на следующем sync-вызове автоматически.
<strong> Cron требует</strong>{" "}
<code>docker compose restart beat</code> после изменения.
</p>
{!cfg.data && <p style={{ color: "#5b6066" }}>Загрузка настроек</p>}
{cfg.data && (
<div style={{ display: "grid", gap: 16 }}>
<label>
<span style={labelStyle}>
📅 Cron-расписание (минута час дом мес дн)
</span>
<input
type="text"
value={cfgDraft.cron_schedule ?? ""}
onChange={(e) =>
setCfgDraft({ ...cfgDraft, cron_schedule: e.target.value })
}
placeholder="0 6 * * tue"
style={{ ...inputStyle, fontFamily: "monospace" }}
/>
<span style={{ fontSize: 11, color: "#5b6066" }}>
Сейчас активно: <code>{cfg.data.cron_schedule}</code>
</span>
</label>
<label>
<span style={labelStyle}>🌍 Группы (csv)</span>
<input
type="text"
value={cfgDraft.groups_csv ?? ""}
onChange={(e) =>
setCfgDraft({ ...cfgDraft, groups_csv: e.target.value })
}
placeholder="Свердловская область,Челябинск,Тюмень,Пермь"
style={{ ...inputStyle, fontFamily: "monospace" }}
/>
<span style={{ fontSize: 11, color: "#5b6066" }}>
Доступные на тарифе: Екатеринбург, Свердловская область,
Свердловск, Верхняя Пышма, Берёзовский, Сысерть, Нижний Тагил,
Каменск-Уральский, Первоуральск, Новоуральск, Челябинск, Тюмень,
Пермь.
</span>
</label>
<div
style={{
display: "grid",
gridTemplateColumns: "repeat(3, 1fr)",
gap: 16,
}}
>
<label>
<span style={labelStyle}>📊 use_ddu</span>
<select
value={String(cfgDraft.use_ddu ?? true)}
onChange={(e) =>
setCfgDraft({
...cfgDraft,
use_ddu: e.target.value === "true",
})
}
style={inputStyle}
>
<option value="true">true (включить)</option>
<option value="false">false (выключить)</option>
</select>
</label>
<label>
<span style={labelStyle}>📊 use_dkp</span>
<select
value={String(cfgDraft.use_dkp ?? true)}
onChange={(e) =>
setCfgDraft({
...cfgDraft,
use_dkp: e.target.value === "true",
})
}
style={inputStyle}
>
<option value="true">true</option>
<option value="false">false</option>
</select>
</label>
<label>
<span style={labelStyle}>
🗓 period_months_back (для corp_sum)
</span>
<input
type="number"
min={1}
max={24}
value={cfgDraft.period_months_back ?? 1}
onChange={(e) =>
setCfgDraft({
...cfgDraft,
period_months_back: Number(e.target.value),
})
}
style={inputStyle}
/>
<span style={{ fontSize: 11, color: "#5b6066" }}>
1 = последний полный месяц
</span>
</label>
<label>
<span style={labelStyle}> inter_group_delay_s</span>
<input
type="number"
min={0}
max={300}
value={cfgDraft.inter_group_delay_s ?? 30}
onChange={(e) =>
setCfgDraft({
...cfgDraft,
inter_group_delay_s: Number(e.target.value),
})
}
style={inputStyle}
/>
<span style={{ fontSize: 11, color: "#5b6066" }}>
пауза между группами (сек)
</span>
</label>
<label>
<span style={labelStyle}>🚦 rate_ms (HTTP-пауза)</span>
<input
type="number"
min={0}
max={10000}
step={100}
value={cfgDraft.rate_ms ?? 3000}
onChange={(e) =>
setCfgDraft({
...cfgDraft,
rate_ms: Number(e.target.value),
})
}
style={inputStyle}
/>
<span style={{ fontSize: 11, color: "#5b6066" }}>
между запросами в client (мс)
</span>
</label>
<label>
<span style={labelStyle}>🔄 retries (на 429/5xx)</span>
<input
type="number"
min={0}
max={5}
value={cfgDraft.retries ?? 2}
onChange={(e) =>
setCfgDraft({
...cfgDraft,
retries: Number(e.target.value),
})
}
style={inputStyle}
/>
</label>
</div>
<div
style={{
display: "flex",
gap: 12,
alignItems: "center",
flexWrap: "wrap",
}}
>
<button
type="button"
onClick={() => saveCfg.mutate(cfgDraft)}
disabled={saveCfg.isPending}
style={primaryBtn}
>
{saveCfg.isPending ? "Сохранение…" : "💾 Сохранить настройки"}
</button>
<button
type="button"
onClick={() => cfg.data && setCfgDraft(cfg.data)}
style={secondaryBtn}
>
Сбросить к актуальному
</button>
{cfgSaved && (
<span style={{ color: "#16a34a", fontSize: 13 }}>
Сохранено. Параметры (кроме cron) применятся со следующим
sync.
</span>
)}
{saveCfg.error && (
<span style={{ color: "#b3261e", fontSize: 13 }}>
Ошибка: {(saveCfg.error as Error).message}
</span>
)}
</div>
{cfg.data.updated_at && (
<p
style={{
fontSize: 11,
color: "#5b6066",
borderTop: "1px solid #e6e8ec",
paddingTop: 8,
margin: 0,
}}
>
Последнее изменение: {formatIso(cfg.data.updated_at)}
{cfg.data.updated_by && ` (by ${cfg.data.updated_by})`}
</p>
)}
</div>
)}
</section>
{/* 🌐 Запустить наш sync */}
<section style={{ ...cardStyle, marginTop: 16 }}>
<h3 style={sectionTitle}>🌐 Запустить наш sync вручную</h3>
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 0 }}>
Дёргает api.objctv.ru с параметрами из настроек выше. Ручной триггер
для немедленного запуска (вне beat-расписания). ~3 минуты на группу ×
N групп.
</p>
<label style={{ display: "block", marginBottom: 12 }}>
<span style={labelStyle}>
Override групп (csv, пусто = взять из настроек)
</span>
<input
type="text"
value={syncGroupsCsv}
onChange={(e) => setSyncGroupsCsv(e.target.value)}
placeholder={
cfg.data?.groups_csv ?? "Свердл.обл,Челябинск,Тюмень,Пермь"
}
style={{ ...inputStyle, fontFamily: "monospace" }}
/>
</label>
<div
style={{
display: "flex",
gap: 12,
alignItems: "center",
flexWrap: "wrap",
}}
>
<button
type="button"
onClick={() => syncOur.mutate()}
disabled={syncOur.isPending}
style={primaryBtn}
>
{syncOur.isPending ? "Запуск…" : "🚀 Запустить наш sync"}
</button>
{lastSyncTaskId && (
<span style={{ fontSize: 13, color: "#5b6066" }}>
task <code>{lastSyncTaskId}</code> в очереди
</span>
)}
{syncOur.error && (
<span style={{ color: "#b3261e", fontSize: 13 }}>
{(syncOur.error as Error).message}
</span>
)}
</div>
</section>
{/* 🛠 Bootstrap ETL — collapsible */}
<section style={{ ...cardStyle, marginTop: 16, background: "#fafafa" }}>
<button
type="button"
onClick={() => setShowEtl(!showEtl)}
style={{
background: "none",
border: "none",
padding: 0,
font: "inherit",
cursor: "pointer",
color: "#374151",
fontSize: 14,
fontWeight: 600,
textAlign: "left" as const,
width: "100%",
}}
>
{showEtl ? "▼" : "▶"} 🛠 Bootstrap ETL: SQLite Антона PG{" "}
<span style={{ fontWeight: 400, fontSize: 12, color: "#9ca3af" }}>
(legacy fallback, в обычной работе не нужен)
</span>
</button>
{showEtl && (
<div style={{ marginTop: 12, display: "grid", gap: 12 }}>
<p style={{ fontSize: 13, color: "#5b6066", margin: 0 }}>
Используется только для одноразового импорта Антоновского SQLite в
нашу PG (например при cold-start). После того как наш sync
отработал этот ETL не нужен.
</p>
<label>
<span style={labelStyle}>Override sqlite_path</span>
<input
type="text"
value={overrideSqlitePath}
onChange={(e) => setOverrideSqlitePath(e.target.value)}
placeholder="/data/anton-sqlite/analysis.db"
style={{ ...inputStyle, fontFamily: "monospace" }}
/>
</label>
<div style={{ display: "flex", gap: 12, alignItems: "center" }}>
<button
type="button"
onClick={() => trigger.mutate()}
disabled={trigger.isPending}
style={secondaryBtn}
>
{trigger.isPending ? "Запуск…" : "Bootstrap ETL"}
</button>
{lastTaskId && (
<span style={{ fontSize: 13, color: "#5b6066" }}>
task <code>{lastTaskId}</code>
</span>
)}
{trigger.error && (
<span style={{ color: "#b3261e", fontSize: 13 }}>
{(trigger.error as Error).message}
</span>
)}
</div>
</div>
)}
</section>
{/* 📜 Runs */}
<section style={{ ...cardStyle, marginTop: 16 }}>
<h3 style={sectionTitle}>
📜 Последние прогоны (top 20, обновляется каждые 5с)
</h3>
<div style={{ overflowX: "auto" }}>
<table
style={{ width: "100%", borderCollapse: "collapse", fontSize: 12 }}
>
<thead style={{ background: "#f9fafb" }}>
<tr>
<th style={th}>run_id</th>
<th style={th}>started</th>
<th style={th}>group</th>
<th style={th}>duration</th>
<th style={{ ...th, textAlign: "right" as const }}>lots</th>
<th style={{ ...th, textAlign: "right" as const }}>
corp×room
</th>
<th style={{ ...th, textAlign: "right" as const }}>mappings</th>
<th style={th}>status</th>
<th style={th}>trigger</th>
<th style={th}>error</th>
</tr>
</thead>
<tbody>
{runs.data?.map((r) => {
const heartbeatLag = r.heartbeat_at
? (Date.now() - new Date(r.heartbeat_at).getTime()) / 1000
: null;
const isStale =
r.status === "running" &&
heartbeatLag !== null &&
heartbeatLag > 300;
const statusColor =
r.status === "done"
? "#16a34a"
: r.status === "running" && !isStale
? "#1d4ed8"
: r.status === "failed" || isStale
? "#b3261e"
: "#6b7280";
return (
<tr key={r.run_id} style={{ borderTop: "1px solid #f3f4f6" }}>
<td style={{ ...td, fontFamily: "monospace" }}>
{r.run_id}
</td>
<td style={td}>{formatIso(r.started_at)}</td>
<td
style={{ ...td, fontFamily: "monospace", fontSize: 11 }}
>
{r.group_name}
</td>
<td style={td}>
{diffSecs(r.started_at, r.finished_at ?? r.heartbeat_at)}
</td>
<td
style={{
...td,
textAlign: "right" as const,
fontFamily: "monospace",
}}
>
{r.rows_lots?.toLocaleString("ru-RU") ?? "—"}
</td>
<td
style={{
...td,
textAlign: "right" as const,
fontFamily: "monospace",
}}
>
{r.rows_corpus_room?.toLocaleString("ru-RU") ?? "—"}
</td>
<td
style={{
...td,
textAlign: "right" as const,
fontFamily: "monospace",
}}
>
{r.rows_mappings?.toLocaleString("ru-RU") ?? "—"}
</td>
<td style={td}>
<span
style={{
display: "inline-block",
padding: "2px 8px",
borderRadius: 4,
background: statusColor + "22",
color: statusColor,
fontWeight: 500,
}}
>
{isStale ? "stale" : r.status}
</span>
</td>
<td style={{ ...td, color: "#6b7280" }}>
{r.triggered_by}
</td>
<td
style={{
...td,
color: "#b3261e",
maxWidth: 240,
overflow: "hidden",
textOverflow: "ellipsis",
whiteSpace: "nowrap",
}}
title={r.error ?? ""}
>
{r.error ?? ""}
</td>
</tr>
);
})}
{runs.data?.length === 0 && (
<tr>
<td
colSpan={10}
style={{
...td,
textAlign: "center" as const,
color: "#6b7280",
padding: 24,
}}
>
Прогонов ещё не было
</td>
</tr>
)}
</tbody>
</table>
</div>
</section>
<ScrapeLogsPanel
scraperType="objective"
title="Журнал Objective ETL"
subtitle="фильтр scraper_type=objective · обновляется каждые 3 сек"
containerStyle={{ marginTop: 16 }}
/>
</>
);
}
// ── Helper component ────────────────────────────────────────────────────────
function Stat({ label, value }: { label: string; value: number }) {
return (
<div>
<div style={{ fontSize: 12, color: "#5b6066", marginBottom: 4 }}>
{label}
</div>
<div style={{ fontFamily: "monospace", fontSize: 22, fontWeight: 600 }}>
{value.toLocaleString("ru-RU")}
</div>
</div>
);
}
// ── Styles ──────────────────────────────────────────────────────────────────
const sectionTitle = {
margin: "0 0 12px",
fontSize: 16,
fontWeight: 600 as const,
};
const primaryBtn = {
padding: "10px 18px",
background: "#1d4ed8",
color: "#fff",
border: "none",
borderRadius: 6,
fontSize: 14,
fontWeight: 600,
cursor: "pointer",
};
const secondaryBtn = {
padding: "8px 14px",
background: "#f3f4f6",
color: "#1f2937",
border: "1px solid #d1d5db",
borderRadius: 6,
fontSize: 13,
cursor: "pointer",
};