add NSPD
This commit is contained in:
lekss361
parent
db6711446a
commit
78b8d7aa00
4 changed files with 1018 additions and 16 deletions
|
|
@ -358,6 +358,214 @@ def list_logs(
|
|||
]
|
||||
|
||||
|
||||
# ── NSPD (cad_quarters / cad_buildings) ─────────────────────────────────────
|
||||
|
||||
|
||||
class TriggerNspdRequest(BaseModel):
|
||||
region_code: int = Field(default=66, ge=1, le=99)
|
||||
limit: int | None = Field(default=None, ge=1, le=2000)
|
||||
rate_ms: int = Field(default=600, ge=200, le=5000)
|
||||
force: bool = False
|
||||
|
||||
|
||||
@router.post("/nspd")
|
||||
def trigger_nspd_sweep(
|
||||
payload: TriggerNspdRequest,
|
||||
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||
) -> dict[str, Any]:
|
||||
_check_token(x_admin_token)
|
||||
from app.workers.tasks.scrape_nspd import (
|
||||
force_release_lock as nspd_force_release_lock,
|
||||
)
|
||||
from app.workers.tasks.scrape_nspd import (
|
||||
scrape_nspd_region,
|
||||
)
|
||||
|
||||
lock_released = False
|
||||
if payload.force:
|
||||
lock_released = nspd_force_release_lock(payload.region_code)
|
||||
|
||||
result = scrape_nspd_region.apply_async(
|
||||
kwargs={
|
||||
"region_code": payload.region_code,
|
||||
"triggered_by": "manual",
|
||||
"limit": payload.limit,
|
||||
"rate_ms": payload.rate_ms,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"region_code": payload.region_code,
|
||||
"limit": payload.limit,
|
||||
"rate_ms": payload.rate_ms,
|
||||
"force": payload.force,
|
||||
"lock_was_released": lock_released,
|
||||
}
|
||||
|
||||
|
||||
class NspdReleaseLockRequest(BaseModel):
|
||||
region_code: int = Field(..., ge=1, le=99)
|
||||
|
||||
|
||||
@router.post("/nspd/release-lock")
|
||||
def nspd_release_lock(
|
||||
payload: NspdReleaseLockRequest,
|
||||
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||
) -> dict[str, Any]:
|
||||
_check_token(x_admin_token)
|
||||
from app.workers.tasks.scrape_nspd import force_release_lock as nspd_force_release_lock
|
||||
|
||||
released = nspd_force_release_lock(payload.region_code)
|
||||
return {"region_code": payload.region_code, "released": released}
|
||||
|
||||
|
||||
@router.get("/nspd/runs")
|
||||
def list_nspd_runs(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||
limit: int = 20,
|
||||
) -> list[dict[str, Any]]:
|
||||
_check_token(x_admin_token)
|
||||
rows = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
SELECT run_id, started_at, finished_at, heartbeat_at,
|
||||
region_code, pending_count,
|
||||
quarters_ok, quarters_failed, buildings_ok,
|
||||
requests_count, waf_429_count,
|
||||
status, error, triggered_by
|
||||
FROM nspd_scrape_runs
|
||||
ORDER BY started_at DESC
|
||||
LIMIT :lim
|
||||
"""
|
||||
),
|
||||
{"lim": min(limit, 100)},
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"run_id": r["run_id"],
|
||||
"started_at": r["started_at"].isoformat() if r["started_at"] else None,
|
||||
"finished_at": r["finished_at"].isoformat() if r["finished_at"] else None,
|
||||
"heartbeat_at": r["heartbeat_at"].isoformat() if r["heartbeat_at"] else None,
|
||||
"region_code": r["region_code"],
|
||||
"pending_count": r["pending_count"],
|
||||
"quarters_ok": r["quarters_ok"],
|
||||
"quarters_failed": r["quarters_failed"],
|
||||
"buildings_ok": r["buildings_ok"],
|
||||
"requests_count": r["requests_count"],
|
||||
"waf_429_count": r["waf_429_count"],
|
||||
"status": r["status"],
|
||||
"error": r["error"],
|
||||
"triggered_by": r["triggered_by"],
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
@router.get("/nspd/logs")
|
||||
def list_nspd_logs(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||
run_id: int | None = None,
|
||||
since_id: int | None = None,
|
||||
limit: int = 200,
|
||||
) -> list[dict[str, Any]]:
|
||||
_check_token(x_admin_token)
|
||||
where: list[str] = []
|
||||
params: dict[str, Any] = {"lim": min(limit, 1000)}
|
||||
if run_id is not None:
|
||||
where.append("run_id = :rid")
|
||||
params["rid"] = run_id
|
||||
if since_id is not None:
|
||||
where.append("log_id > :sid")
|
||||
params["sid"] = since_id
|
||||
where_sql = ("WHERE " + " AND ".join(where)) if where else ""
|
||||
rows = (
|
||||
db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT log_id, run_id, ts, level, stage, cad_number, message
|
||||
FROM nspd_scrape_log
|
||||
{where_sql}
|
||||
ORDER BY log_id DESC
|
||||
LIMIT :lim
|
||||
"""
|
||||
),
|
||||
params,
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"log_id": r["log_id"],
|
||||
"run_id": r["run_id"],
|
||||
"ts": r["ts"].isoformat() if r["ts"] else None,
|
||||
"level": r["level"],
|
||||
"stage": r["stage"],
|
||||
"cad_number": r["cad_number"],
|
||||
"message": r["message"],
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
@router.get("/nspd/coverage")
|
||||
def nspd_coverage(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||
region_code: int = 66,
|
||||
) -> dict[str, Any]:
|
||||
"""Покрытие cad_quarters_geom относительно ДДУ-сделок rosreestr.
|
||||
Полезно перед запуском sweep — видишь сколько ещё осталось скрейпить."""
|
||||
_check_token(x_admin_token)
|
||||
row = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
WITH pending AS (
|
||||
SELECT DISTINCT quarter_cad_number AS cad
|
||||
FROM rosreestr_deals
|
||||
WHERE region_code = :rc
|
||||
AND doc_type = 'ДДУ'
|
||||
AND realestate_type_code = '002001003000'
|
||||
AND quarter_cad_number IS NOT NULL
|
||||
AND quarter_cad_number !~ '^00:00:'
|
||||
AND quarter_cad_number !~ ':0000000$'
|
||||
),
|
||||
done AS (SELECT cad_number AS cad FROM cad_quarters_geom)
|
||||
SELECT
|
||||
(SELECT COUNT(*) FROM pending) AS total_with_ddu,
|
||||
(SELECT COUNT(*) FROM pending p
|
||||
WHERE EXISTS (SELECT 1 FROM done d WHERE d.cad = p.cad))
|
||||
AS already_scraped,
|
||||
(SELECT COUNT(*) FROM cad_buildings) AS buildings_total
|
||||
"""
|
||||
),
|
||||
{"rc": region_code},
|
||||
)
|
||||
.mappings()
|
||||
.first()
|
||||
)
|
||||
total = int(row["total_with_ddu"] or 0)
|
||||
done = int(row["already_scraped"] or 0)
|
||||
return {
|
||||
"region_code": region_code,
|
||||
"total_quarters_with_ddu": total,
|
||||
"already_scraped": done,
|
||||
"pending": max(total - done, 0),
|
||||
"buildings_total": int(row["buildings_total"] or 0),
|
||||
"coverage_pct": round(done / max(total, 1) * 100, 1) if total else None,
|
||||
}
|
||||
|
||||
|
||||
# ── DomRF (kn) — runs list ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
@router.get("/runs")
|
||||
def list_runs(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
|
|
|
|||
|
|
@ -149,25 +149,46 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None:
|
|||
logger.warning("worker_ready: failed to enqueue resume for run=%s: %s", rid, e)
|
||||
|
||||
# NSPD-runs: помечаем зомби (>15 мин без heartbeat — NSPD сам по себе
|
||||
# медленнее kn, дольше дельта). Resume не делаем — следующий beat
|
||||
# подберёт pending (idempotent через cad_quarters_geom).
|
||||
# медленнее kn, дельта больше). Resume = просто заново ставим
|
||||
# scrape_nspd_region с тем же region_code и triggered_by='resume':
|
||||
# сервис идемпотентен — pending = (rosreestr ДДУ cads) − (cad_quarters_geom done),
|
||||
# уже скрейпнутые квартала пропустятся автоматом.
|
||||
db = SessionLocal()
|
||||
nspd_resume_targets: list[int] = []
|
||||
try:
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE nspd_scrape_runs
|
||||
SET status = 'zombie',
|
||||
finished_at = NOW(),
|
||||
error = COALESCE(error,
|
||||
'auto-zombie at worker_ready')
|
||||
WHERE status = 'running'
|
||||
AND COALESCE(heartbeat_at, started_at)
|
||||
< NOW() - INTERVAL '15 minutes'
|
||||
"""
|
||||
zombie_rows = (
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
UPDATE nspd_scrape_runs
|
||||
SET status = 'zombie',
|
||||
finished_at = NOW(),
|
||||
error = COALESCE(error,
|
||||
'auto-zombie at worker_ready, resume scheduled')
|
||||
WHERE status = 'running'
|
||||
AND COALESCE(heartbeat_at, started_at)
|
||||
< NOW() - INTERVAL '15 minutes'
|
||||
RETURNING run_id, region_code
|
||||
"""
|
||||
)
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
db.commit()
|
||||
# Уникальные регионы для resume (разные ран-id для одного региона —
|
||||
# повторно не enqueueum, lock и idempotency пропустят дубль).
|
||||
seen: set[int] = set()
|
||||
for r in zombie_rows:
|
||||
rc = int(r["region_code"])
|
||||
if rc not in seen:
|
||||
seen.add(rc)
|
||||
nspd_resume_targets.append(rc)
|
||||
logger.info(
|
||||
"worker_ready: NSPD zombie run=%s region=%s — resume scheduled",
|
||||
r["run_id"],
|
||||
rc,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("worker_ready nspd zombie scan failed: %s", e)
|
||||
try:
|
||||
|
|
@ -176,3 +197,19 @@ def _resume_zombie_runs(sender=None, **_kwargs) -> None:
|
|||
pass
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
if nspd_resume_targets:
|
||||
from app.workers.tasks.scrape_nspd import scrape_nspd_region
|
||||
|
||||
for rc in nspd_resume_targets:
|
||||
try:
|
||||
scrape_nspd_region.apply_async(
|
||||
kwargs={"region_code": rc, "triggered_by": "resume"},
|
||||
)
|
||||
logger.info("worker_ready: scrape_nspd_region enqueued region=%s", rc)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"worker_ready: failed to enqueue NSPD resume for region=%s: %s",
|
||||
rc,
|
||||
e,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,8 @@ import Link from "next/link";
|
|||
import { usePathname } from "next/navigation";
|
||||
|
||||
const TABS = [
|
||||
{ href: "/admin/scrape", label: "Скрапер" },
|
||||
{ href: "/admin/scrape", label: "Скрапер DOM.РФ" },
|
||||
{ href: "/admin/scrape/nspd", label: "Скрапер NSPD" },
|
||||
{ href: "/admin/leads", label: "Лиды PRINZIP" },
|
||||
];
|
||||
|
||||
|
|
@ -35,8 +36,19 @@ export default function AdminLayout({
|
|||
}}
|
||||
>
|
||||
{TABS.map((t) => {
|
||||
const active =
|
||||
// Активна вкладка с самым длинным совпадающим префиксом —
|
||||
// иначе /admin/scrape тоже подсветится при заходе на
|
||||
// /admin/scrape/nspd.
|
||||
const matchesPrefix =
|
||||
pathname === t.href || pathname?.startsWith(t.href + "/");
|
||||
const longerMatch = TABS.some(
|
||||
(other) =>
|
||||
other.href !== t.href &&
|
||||
other.href.length > t.href.length &&
|
||||
(pathname === other.href ||
|
||||
pathname?.startsWith(other.href + "/")),
|
||||
);
|
||||
const active = matchesPrefix && !longerMatch;
|
||||
return (
|
||||
<Link
|
||||
key={t.href}
|
||||
|
|
|
|||
745
frontend/src/app/admin/scrape/nspd/page.tsx
Normal file
745
frontend/src/app/admin/scrape/nspd/page.tsx
Normal file
|
|
@ -0,0 +1,745 @@
|
|||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
|
||||
import { apiFetch } from "@/lib/api";
|
||||
|
||||
interface NspdRunRow {
|
||||
run_id: number;
|
||||
started_at: string | null;
|
||||
finished_at: string | null;
|
||||
heartbeat_at: string | null;
|
||||
region_code: number;
|
||||
pending_count: number | null;
|
||||
quarters_ok: number;
|
||||
quarters_failed: number;
|
||||
buildings_ok: number;
|
||||
requests_count: number;
|
||||
waf_429_count: number;
|
||||
status: string;
|
||||
error: string | null;
|
||||
triggered_by: string;
|
||||
}
|
||||
|
||||
interface NspdLogRow {
|
||||
log_id: number;
|
||||
run_id: number | null;
|
||||
ts: string | null;
|
||||
level: "debug" | "info" | "warn" | "error";
|
||||
stage: string | null;
|
||||
cad_number: string | null;
|
||||
message: string;
|
||||
}
|
||||
|
||||
interface NspdCoverage {
|
||||
region_code: number;
|
||||
total_quarters_with_ddu: number;
|
||||
already_scraped: number;
|
||||
pending: number;
|
||||
buildings_total: number;
|
||||
coverage_pct: number | null;
|
||||
}
|
||||
|
||||
interface NspdTriggerResp {
|
||||
task_id: string;
|
||||
region_code: number;
|
||||
limit: number | null;
|
||||
rate_ms: number;
|
||||
force: boolean;
|
||||
lock_was_released: boolean;
|
||||
}
|
||||
|
||||
export default function NspdScrapeAdminPage() {
|
||||
const [token, setToken] = useState<string>(() =>
|
||||
typeof window === "undefined"
|
||||
? ""
|
||||
: (localStorage.getItem("admin_token") ?? ""),
|
||||
);
|
||||
const [region, setRegion] = useState(66);
|
||||
const [limit, setLimit] = useState<string>("");
|
||||
const [rateMs, setRateMs] = useState(600);
|
||||
const [forceRun, setForceRun] = useState(false);
|
||||
const [logsRunId, setLogsRunId] = useState<number | "">("");
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const saveToken = (v: string) => {
|
||||
setToken(v);
|
||||
localStorage.setItem("admin_token", v);
|
||||
};
|
||||
|
||||
const coverage = useQuery({
|
||||
queryKey: ["nspd-coverage", region, token],
|
||||
queryFn: () =>
|
||||
apiFetch<NspdCoverage>(
|
||||
`/api/v1/admin/scrape/nspd/coverage?region_code=${region}`,
|
||||
{
|
||||
headers: { "X-Admin-Token": token },
|
||||
},
|
||||
),
|
||||
enabled: !!token,
|
||||
refetchInterval: 10000,
|
||||
});
|
||||
|
||||
const runs = useQuery({
|
||||
queryKey: ["nspd-runs", token],
|
||||
queryFn: () =>
|
||||
apiFetch<NspdRunRow[]>("/api/v1/admin/scrape/nspd/runs?limit=20", {
|
||||
headers: { "X-Admin-Token": token },
|
||||
}),
|
||||
enabled: !!token,
|
||||
refetchInterval: 5000,
|
||||
});
|
||||
|
||||
const logs = useQuery({
|
||||
queryKey: ["nspd-logs", logsRunId, token],
|
||||
queryFn: () => {
|
||||
const q =
|
||||
logsRunId !== "" ? `?run_id=${logsRunId}&limit=200` : "?limit=200";
|
||||
return apiFetch<NspdLogRow[]>(`/api/v1/admin/scrape/nspd/logs${q}`, {
|
||||
headers: { "X-Admin-Token": token },
|
||||
});
|
||||
},
|
||||
enabled: !!token,
|
||||
refetchInterval: 3000,
|
||||
});
|
||||
|
||||
const trigger = useMutation({
|
||||
mutationFn: () =>
|
||||
apiFetch<NspdTriggerResp>("/api/v1/admin/scrape/nspd", {
|
||||
method: "POST",
|
||||
headers: { "X-Admin-Token": token },
|
||||
body: JSON.stringify({
|
||||
region_code: region,
|
||||
limit: limit === "" ? null : Number(limit),
|
||||
rate_ms: rateMs,
|
||||
force: forceRun,
|
||||
}),
|
||||
}),
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ["nspd-runs"] });
|
||||
queryClient.invalidateQueries({ queryKey: ["nspd-coverage"] });
|
||||
},
|
||||
});
|
||||
|
||||
const releaseLock = useMutation({
|
||||
mutationFn: () =>
|
||||
apiFetch<{ region_code: number; released: boolean }>(
|
||||
"/api/v1/admin/scrape/nspd/release-lock",
|
||||
{
|
||||
method: "POST",
|
||||
headers: { "X-Admin-Token": token },
|
||||
body: JSON.stringify({ region_code: region }),
|
||||
},
|
||||
),
|
||||
onSuccess: () => {
|
||||
queryClient.invalidateQueries({ queryKey: ["nspd-runs"] });
|
||||
},
|
||||
});
|
||||
|
||||
return (
|
||||
<>
|
||||
<h2 style={{ marginTop: 0, fontSize: 18 }}>
|
||||
Скрапер NSPD — кадастровые кварталы и здания
|
||||
</h2>
|
||||
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 4 }}>
|
||||
Берёт quarter_cad_number из rosreestr_deals (ДДУ-сделки на квартиры) →
|
||||
тянет полигон cad-квартала + здания внутри из NSPD geoportal. Расписание
|
||||
(beat): 20-е число фев/мая/авг/нояб (после квартальных публикаций
|
||||
rosreestr). См. <code>SCRAPE_NSPD_CRON</code>.
|
||||
</p>
|
||||
|
||||
<section style={cardStyle}>
|
||||
<label style={{ display: "block", marginBottom: 12 }}>
|
||||
<span style={labelStyle}>Admin Token</span>
|
||||
<input
|
||||
type="password"
|
||||
value={token}
|
||||
onChange={(e) => saveToken(e.target.value)}
|
||||
placeholder="SCRAPE_ADMIN_TOKEN"
|
||||
style={inputStyle}
|
||||
/>
|
||||
</label>
|
||||
<div style={{ display: "flex", gap: 16, flexWrap: "wrap" }}>
|
||||
<label>
|
||||
<span style={labelStyle}>Регион</span>
|
||||
<input
|
||||
type="number"
|
||||
value={region}
|
||||
min={1}
|
||||
max={99}
|
||||
onChange={(e) => setRegion(Number(e.target.value))}
|
||||
style={{ ...inputStyle, width: 100 }}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
<span style={labelStyle}>Limit (smoke)</span>
|
||||
<input
|
||||
type="number"
|
||||
value={limit}
|
||||
onChange={(e) => setLimit(e.target.value)}
|
||||
placeholder="напр. 10"
|
||||
min={1}
|
||||
max={2000}
|
||||
style={{ ...inputStyle, width: 120 }}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
<span style={labelStyle}>Rate (мс)</span>
|
||||
<input
|
||||
type="number"
|
||||
value={rateMs}
|
||||
onChange={(e) => setRateMs(Number(e.target.value))}
|
||||
min={200}
|
||||
max={5000}
|
||||
step={100}
|
||||
style={{ ...inputStyle, width: 110 }}
|
||||
title="Пауза между NSPD-запросами. <600мс = WAF 403 риск"
|
||||
/>
|
||||
</label>
|
||||
<label
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
marginTop: 22,
|
||||
}}
|
||||
title="Перед запуском удалить existing Redis-lock"
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={forceRun}
|
||||
onChange={(e) => setForceRun(e.target.checked)}
|
||||
/>
|
||||
<span style={{ color: forceRun ? "#b3261e" : "inherit" }}>
|
||||
⚡ Force (снять lock)
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{/* Pre-launch summary: сколько кварталов реально пойдёт в обработку
|
||||
и сколько примерно займёт. Считаем тут, а не на бэке — дешевле. */}
|
||||
{token && coverage.data
|
||||
? (() => {
|
||||
const limitNum = limit === "" ? null : Number(limit);
|
||||
const willProcess = limitNum
|
||||
? Math.min(limitNum, coverage.data.pending)
|
||||
: coverage.data.pending;
|
||||
// 2 запроса на квартал (полигон + здания) × rate_ms, +1с на запись
|
||||
const etaSec = willProcess * ((rateMs * 2) / 1000 + 1);
|
||||
const etaMin = Math.round(etaSec / 60);
|
||||
const etaH = Math.floor(etaMin / 60);
|
||||
const etaText =
|
||||
etaMin >= 60 ? `~${etaH}ч ${etaMin % 60}мин` : `~${etaMin} мин`;
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
marginTop: 14,
|
||||
padding: 10,
|
||||
background: willProcess === 0 ? "#f0fdf4" : "#eff6ff",
|
||||
border: `1px solid ${
|
||||
willProcess === 0 ? "#bbf7d0" : "#bfdbfe"
|
||||
}`,
|
||||
borderRadius: 6,
|
||||
fontSize: 13,
|
||||
lineHeight: 1.5,
|
||||
}}
|
||||
>
|
||||
{willProcess === 0 ? (
|
||||
<>
|
||||
✅ Всё уже скрейпнуто для региона {region}. Запускать
|
||||
бесполезно — pending=0.
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
При запуске будет обработано{" "}
|
||||
<strong>
|
||||
{willProcess.toLocaleString("ru")} кварталов
|
||||
</strong>{" "}
|
||||
{limitNum && limitNum < coverage.data.pending ? (
|
||||
<span style={{ color: "#5b6066" }}>
|
||||
(limit {limitNum} из {coverage.data.pending} pending)
|
||||
</span>
|
||||
) : (
|
||||
<span style={{ color: "#5b6066" }}>
|
||||
(вся очередь pending)
|
||||
</span>
|
||||
)}
|
||||
. Расчётное время: <strong>{etaText}</strong> при rate{" "}
|
||||
{rateMs}мс ({((rateMs * 2) / 1000).toFixed(1)}с / квартал
|
||||
× 2 запроса).
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
})()
|
||||
: null}
|
||||
|
||||
<div
|
||||
style={{ marginTop: 16, display: "flex", gap: 12, flexWrap: "wrap" }}
|
||||
>
|
||||
<button
|
||||
onClick={() => trigger.mutate()}
|
||||
disabled={
|
||||
!token || trigger.isPending || (coverage.data?.pending ?? 0) === 0
|
||||
}
|
||||
style={primaryBtn}
|
||||
>
|
||||
{trigger.isPending ? "Запускаю…" : "Запустить sweep"}
|
||||
</button>
|
||||
<button
|
||||
onClick={() => {
|
||||
if (
|
||||
confirm(
|
||||
"Снять Redis-lock для NSPD? Используй когда подозреваешь зомби-worker.",
|
||||
)
|
||||
) {
|
||||
releaseLock.mutate();
|
||||
}
|
||||
}}
|
||||
disabled={!token || releaseLock.isPending}
|
||||
style={secondaryBtn}
|
||||
>
|
||||
{releaseLock.isPending ? "Снимаю…" : "🔓 Снять Redis-lock"}
|
||||
</button>
|
||||
{!token ? (
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 12,
|
||||
color: "#9a6700",
|
||||
fontSize: 13,
|
||||
alignSelf: "center",
|
||||
}}
|
||||
>
|
||||
← введите Admin Token выше
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
|
||||
{trigger.isError ? (
|
||||
<div
|
||||
style={{
|
||||
marginTop: 12,
|
||||
padding: 12,
|
||||
background: "#fef2f2",
|
||||
border: "1px solid #fecaca",
|
||||
borderRadius: 6,
|
||||
color: "#991b1b",
|
||||
fontSize: 14,
|
||||
wordBreak: "break-word",
|
||||
}}
|
||||
>
|
||||
<strong>Ошибка:</strong> {String(trigger.error)}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{trigger.isSuccess ? (
|
||||
<div
|
||||
style={{
|
||||
marginTop: 12,
|
||||
padding: 12,
|
||||
background: "#f0fdf4",
|
||||
border: "1px solid #bbf7d0",
|
||||
borderRadius: 6,
|
||||
color: "#065f46",
|
||||
fontSize: 14,
|
||||
}}
|
||||
>
|
||||
<strong>✓ Поставлено в очередь.</strong> task_id={" "}
|
||||
<code
|
||||
style={{
|
||||
background: "#dcfce7",
|
||||
padding: "2px 6px",
|
||||
borderRadius: 3,
|
||||
}}
|
||||
>
|
||||
{trigger.data.task_id}
|
||||
</code>{" "}
|
||||
· регион {trigger.data.region_code}
|
||||
{trigger.data.limit ? ` · limit=${trigger.data.limit}` : ""} · rate=
|
||||
{trigger.data.rate_ms}мс
|
||||
</div>
|
||||
) : null}
|
||||
</section>
|
||||
|
||||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||
<h2 style={{ margin: 0, fontSize: 16, marginBottom: 12 }}>
|
||||
Покрытие cad_quarters_geom (регион {region})
|
||||
</h2>
|
||||
{!token ? (
|
||||
<p style={{ color: "#5b6066", fontSize: 13 }}>
|
||||
Введите Admin Token выше.
|
||||
</p>
|
||||
) : coverage.isLoading ? (
|
||||
<p style={{ color: "#5b6066" }}>Загрузка…</p>
|
||||
) : coverage.isError ? (
|
||||
<p style={{ color: "#b3261e" }}>Ошибка: {String(coverage.error)}</p>
|
||||
) : coverage.data ? (
|
||||
<div
|
||||
style={{
|
||||
display: "grid",
|
||||
gridTemplateColumns: "repeat(auto-fit, minmax(160px, 1fr))",
|
||||
gap: 12,
|
||||
}}
|
||||
>
|
||||
<CoverageCard
|
||||
label="ДДУ-кварталов в rosreestr"
|
||||
value={coverage.data.total_quarters_with_ddu.toLocaleString("ru")}
|
||||
/>
|
||||
<CoverageCard
|
||||
label="Уже скрейпнуто"
|
||||
value={coverage.data.already_scraped.toLocaleString("ru")}
|
||||
color="#0a7a3a"
|
||||
/>
|
||||
<CoverageCard
|
||||
label="Осталось"
|
||||
value={coverage.data.pending.toLocaleString("ru")}
|
||||
color={coverage.data.pending > 0 ? "#9a6700" : "#0a7a3a"}
|
||||
/>
|
||||
<CoverageCard
|
||||
label="Зданий в БД"
|
||||
value={coverage.data.buildings_total.toLocaleString("ru")}
|
||||
/>
|
||||
<CoverageCard
|
||||
label="Покрытие, %"
|
||||
value={
|
||||
coverage.data.coverage_pct != null
|
||||
? `${coverage.data.coverage_pct}%`
|
||||
: "—"
|
||||
}
|
||||
color={
|
||||
coverage.data.coverage_pct != null &&
|
||||
coverage.data.coverage_pct >= 90
|
||||
? "#0a7a3a"
|
||||
: "#9a6700"
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
) : null}
|
||||
</section>
|
||||
|
||||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "space-between",
|
||||
marginBottom: 12,
|
||||
}}
|
||||
>
|
||||
<h2 style={{ margin: 0, fontSize: 16 }}>
|
||||
Лог скрейпа{" "}
|
||||
<span style={{ fontWeight: 400, color: "#5b6066", fontSize: 13 }}>
|
||||
· обновляется каждые 3 сек
|
||||
</span>
|
||||
</h2>
|
||||
<input
|
||||
type="number"
|
||||
value={logsRunId}
|
||||
onChange={(e) =>
|
||||
setLogsRunId(e.target.value === "" ? "" : Number(e.target.value))
|
||||
}
|
||||
placeholder="run_id (опц.)"
|
||||
style={{ ...inputStyle, width: 130 }}
|
||||
/>
|
||||
</div>
|
||||
{!token ? (
|
||||
<p style={{ color: "#5b6066", fontSize: 13 }}>
|
||||
Введите Admin Token чтобы видеть лог.
|
||||
</p>
|
||||
) : logs.isLoading ? (
|
||||
<p style={{ color: "#5b6066" }}>Загрузка…</p>
|
||||
) : logs.isError ? (
|
||||
<p style={{ color: "#b3261e" }}>Ошибка: {String(logs.error)}</p>
|
||||
) : (logs.data?.length ?? 0) === 0 ? (
|
||||
<p style={{ color: "#5b6066", fontSize: 13 }}>
|
||||
Нет событий{logsRunId !== "" ? ` для run_id=${logsRunId}` : ""}.
|
||||
</p>
|
||||
) : (
|
||||
<div
|
||||
style={{
|
||||
maxHeight: 360,
|
||||
overflowY: "auto",
|
||||
fontFamily: "ui-monospace, SFMono-Regular, Menlo, monospace",
|
||||
fontSize: 12,
|
||||
border: "1px solid #eef0f3",
|
||||
borderRadius: 6,
|
||||
background: "#0f172a",
|
||||
color: "#e2e8f0",
|
||||
padding: 10,
|
||||
}}
|
||||
>
|
||||
{(logs.data ?? []).map((l) => {
|
||||
const color =
|
||||
l.level === "error"
|
||||
? "#fca5a5"
|
||||
: l.level === "warn"
|
||||
? "#fcd34d"
|
||||
: "#a3e635";
|
||||
return (
|
||||
<div key={l.log_id} style={{ marginBottom: 2 }}>
|
||||
<span style={{ color: "#94a3b8" }}>
|
||||
{l.ts?.slice(11, 19)}
|
||||
</span>{" "}
|
||||
<span style={{ color }}>{l.level.toUpperCase()}</span>{" "}
|
||||
<span style={{ color: "#7dd3fc" }}>
|
||||
[run-{l.run_id}
|
||||
{l.stage ? ` · ${l.stage}` : ""}
|
||||
{l.cad_number ? ` · ${l.cad_number}` : ""}]
|
||||
</span>{" "}
|
||||
<span>{l.message}</span>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||
<h2 style={{ marginTop: 0, fontSize: 16 }}>Последние прогоны</h2>
|
||||
{!token ? (
|
||||
<p style={{ color: "#5b6066" }}>
|
||||
Введите Admin Token чтобы увидеть журнал.
|
||||
</p>
|
||||
) : runs.isLoading ? (
|
||||
<p>Загрузка…</p>
|
||||
) : runs.isError ? (
|
||||
<p style={{ color: "#b3261e" }}>Ошибка: {String(runs.error)}</p>
|
||||
) : (
|
||||
<table
|
||||
style={{ width: "100%", borderCollapse: "collapse", fontSize: 13 }}
|
||||
>
|
||||
<thead>
|
||||
<tr style={{ background: "#f6f7f9" }}>
|
||||
{[
|
||||
"#",
|
||||
"Старт",
|
||||
"HB",
|
||||
"Статус",
|
||||
"Прогресс",
|
||||
"Pending",
|
||||
"OK",
|
||||
"Fail",
|
||||
"Зданий",
|
||||
"Req",
|
||||
"WAF",
|
||||
"Trigger",
|
||||
"Ошибка",
|
||||
].map((h) => (
|
||||
<th key={h} style={th}>
|
||||
{h}
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(runs.data ?? []).map((r) => (
|
||||
<tr
|
||||
key={r.run_id}
|
||||
style={{ borderBottom: "1px solid #eef0f3" }}
|
||||
>
|
||||
<td style={td}>{r.run_id}</td>
|
||||
<td style={td}>
|
||||
{r.started_at?.slice(0, 19).replace("T", " ")}
|
||||
</td>
|
||||
<td style={td}>
|
||||
{r.heartbeat_at ? r.heartbeat_at.slice(11, 19) : "—"}
|
||||
</td>
|
||||
<td
|
||||
style={{
|
||||
...td,
|
||||
color:
|
||||
r.status === "done"
|
||||
? "#0a7a3a"
|
||||
: r.status === "failed" || r.status === "zombie"
|
||||
? "#b3261e"
|
||||
: "#9a6700",
|
||||
fontWeight: 600,
|
||||
}}
|
||||
>
|
||||
{r.status}
|
||||
</td>
|
||||
<td style={{ ...td, minWidth: 110 }}>
|
||||
{r.pending_count && r.pending_count > 0
|
||||
? (() => {
|
||||
const pct = Math.min(
|
||||
100,
|
||||
Math.round(
|
||||
((r.quarters_ok + r.quarters_failed) /
|
||||
r.pending_count) *
|
||||
100,
|
||||
),
|
||||
);
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
fontFamily: "ui-monospace, monospace",
|
||||
fontSize: 12,
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: 70,
|
||||
height: 6,
|
||||
background: "#eef0f3",
|
||||
borderRadius: 3,
|
||||
overflow: "hidden",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
width: `${pct}%`,
|
||||
height: "100%",
|
||||
background:
|
||||
pct === 100
|
||||
? "#0a7a3a"
|
||||
: pct > 0
|
||||
? "#1d4ed8"
|
||||
: "#9a6700",
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
<span>{pct}%</span>
|
||||
</div>
|
||||
);
|
||||
})()
|
||||
: "—"}
|
||||
</td>
|
||||
<td style={td}>{r.pending_count ?? "—"}</td>
|
||||
<td style={td}>{r.quarters_ok}</td>
|
||||
<td
|
||||
style={{
|
||||
...td,
|
||||
color: r.quarters_failed > 0 ? "#b3261e" : undefined,
|
||||
}}
|
||||
>
|
||||
{r.quarters_failed}
|
||||
</td>
|
||||
<td style={td}>{r.buildings_ok}</td>
|
||||
<td style={td}>{r.requests_count}</td>
|
||||
<td
|
||||
style={{
|
||||
...td,
|
||||
color: r.waf_429_count > 0 ? "#b3261e" : undefined,
|
||||
}}
|
||||
>
|
||||
{r.waf_429_count}
|
||||
</td>
|
||||
<td style={td}>{r.triggered_by}</td>
|
||||
<td
|
||||
style={{
|
||||
...td,
|
||||
color: "#b3261e",
|
||||
maxWidth: 240,
|
||||
overflow: "hidden",
|
||||
textOverflow: "ellipsis",
|
||||
whiteSpace: "nowrap",
|
||||
}}
|
||||
title={r.error ?? ""}
|
||||
>
|
||||
{r.error ?? ""}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</section>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
function CoverageCard({
|
||||
label,
|
||||
value,
|
||||
color,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
color?: string;
|
||||
}) {
|
||||
return (
|
||||
<div
|
||||
style={{
|
||||
background: "#fff",
|
||||
border: "1px solid #e6e8ec",
|
||||
borderRadius: 8,
|
||||
padding: "10px 12px",
|
||||
}}
|
||||
>
|
||||
<div
|
||||
style={{
|
||||
fontSize: 11,
|
||||
color: "#5b6066",
|
||||
textTransform: "uppercase",
|
||||
letterSpacing: 0.4,
|
||||
marginBottom: 4,
|
||||
}}
|
||||
>
|
||||
{label}
|
||||
</div>
|
||||
<div
|
||||
style={{
|
||||
fontSize: 22,
|
||||
fontWeight: 700,
|
||||
color: color ?? "#0f172a",
|
||||
lineHeight: 1.1,
|
||||
}}
|
||||
>
|
||||
{value}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const cardStyle = {
|
||||
background: "#fff",
|
||||
border: "1px solid #e6e8ec",
|
||||
borderRadius: 12,
|
||||
padding: 20,
|
||||
};
|
||||
const labelStyle = {
|
||||
display: "block",
|
||||
fontSize: 12,
|
||||
color: "#5b6066",
|
||||
marginBottom: 4,
|
||||
textTransform: "uppercase" as const,
|
||||
letterSpacing: 0.4,
|
||||
};
|
||||
const inputStyle = {
|
||||
padding: "8px 10px",
|
||||
border: "1px solid #d1d5db",
|
||||
borderRadius: 6,
|
||||
fontSize: 14,
|
||||
width: "100%",
|
||||
boxSizing: "border-box" as const,
|
||||
};
|
||||
const primaryBtn = {
|
||||
padding: "10px 18px",
|
||||
background: "#1d4ed8",
|
||||
color: "#fff",
|
||||
border: "none",
|
||||
borderRadius: 6,
|
||||
fontSize: 14,
|
||||
fontWeight: 600,
|
||||
cursor: "pointer",
|
||||
};
|
||||
const secondaryBtn = {
|
||||
padding: "8px 14px",
|
||||
background: "#f3f4f6",
|
||||
color: "#1f2937",
|
||||
border: "1px solid #d1d5db",
|
||||
borderRadius: 6,
|
||||
fontSize: 13,
|
||||
cursor: "pointer",
|
||||
};
|
||||
const th = {
|
||||
padding: "8px 10px",
|
||||
textAlign: "left" as const,
|
||||
fontWeight: 600,
|
||||
borderBottom: "1px solid #e6e8ec",
|
||||
};
|
||||
const td = { padding: "8px 10px" };
|
||||
Loading…
Add table
Reference in a new issue