feat(22d): admin UI buttons (skip/force) + reduce catalog jitter 800→300ms #352
2 changed files with 143 additions and 2 deletions
|
|
@ -190,7 +190,7 @@ async def fetch_catalog_object_html(session: BrowserSession, obj_id: int) -> str
|
||||||
|
|
||||||
for attempt in range(5):
|
for attempt in range(5):
|
||||||
async with session._sem:
|
async with session._sem:
|
||||||
await jitter_sleep(800, 1500)
|
await jitter_sleep(300, 700)
|
||||||
try:
|
try:
|
||||||
session._request_count += 1
|
session._request_count += 1
|
||||||
result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url})
|
result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url})
|
||||||
|
|
@ -412,7 +412,7 @@ async def scrape_catalog_objects(
|
||||||
) -> dict[str, int]:
|
) -> dict[str, int]:
|
||||||
"""Scrape списка объектов через один BrowserSession.
|
"""Scrape списка объектов через один BrowserSession.
|
||||||
|
|
||||||
Запускает один BrowserSession на весь batch; jitter_sleep (800–1500 мс)
|
Запускает один BrowserSession на весь batch; jitter_sleep (300–700 мс)
|
||||||
встроен в fetch_catalog_object_html для защиты от rate-limit.
|
встроен в fetch_catalog_object_html для защиты от rate-limit.
|
||||||
|
|
||||||
Returns:
|
Returns:
|
||||||
|
|
|
||||||
|
|
@ -26,6 +26,13 @@ interface TriggerResp {
|
||||||
developers: string[] | null;
|
developers: string[] | null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
interface CatalogTriggerResp {
|
||||||
|
task_id: string;
|
||||||
|
region_code: number;
|
||||||
|
max_objects: number | null;
|
||||||
|
force: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
interface QueueTask {
|
interface QueueTask {
|
||||||
worker: string;
|
worker: string;
|
||||||
task_id: string;
|
task_id: string;
|
||||||
|
|
@ -78,6 +85,7 @@ export default function ScrapeAdminPage() {
|
||||||
const [extras, setExtras] = useState(true);
|
const [extras, setExtras] = useState(true);
|
||||||
const [downloadPhotos, setDownloadPhotos] = useState(false);
|
const [downloadPhotos, setDownloadPhotos] = useState(false);
|
||||||
const [forceRun, setForceRun] = useState(false);
|
const [forceRun, setForceRun] = useState(false);
|
||||||
|
const [catalogMaxObjects, setCatalogMaxObjects] = useState<number | "">("");
|
||||||
const [showFailures, setShowFailures] = useState(false);
|
const [showFailures, setShowFailures] = useState(false);
|
||||||
const [failuresRunId, setFailuresRunId] = useState<number | "">("");
|
const [failuresRunId, setFailuresRunId] = useState<number | "">("");
|
||||||
const [logsRunId, setLogsRunId] = useState<number | "">("");
|
const [logsRunId, setLogsRunId] = useState<number | "">("");
|
||||||
|
|
@ -121,6 +129,27 @@ export default function ScrapeAdminPage() {
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const catalogTrigger = useMutation({
|
||||||
|
mutationFn: (force: boolean) =>
|
||||||
|
apiFetch<CatalogTriggerResp>("/api/v1/admin/scrape/kn-catalog-objects", {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "X-Admin-Token": token },
|
||||||
|
body: JSON.stringify({
|
||||||
|
region_code: region,
|
||||||
|
max_objects: catalogMaxObjects === "" ? null : catalogMaxObjects,
|
||||||
|
force,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
onSuccess: (data) => {
|
||||||
|
console.log("[scrape] catalog queued:", data);
|
||||||
|
queryClient.invalidateQueries({ queryKey: ["scrape-runs"] });
|
||||||
|
queryClient.invalidateQueries({ queryKey: ["scrape-queue"] });
|
||||||
|
},
|
||||||
|
onError: (err) => {
|
||||||
|
console.error("[scrape] catalog failed:", err);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
const releaseLock = useMutation({
|
const releaseLock = useMutation({
|
||||||
mutationFn: () =>
|
mutationFn: () =>
|
||||||
apiFetch<{
|
apiFetch<{
|
||||||
|
|
@ -458,6 +487,118 @@ export default function ScrapeAdminPage() {
|
||||||
) : null}
|
) : null}
|
||||||
</section>
|
</section>
|
||||||
|
|
||||||
|
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||||
|
<h2 style={{ margin: 0, fontSize: 16 }}>📦 Catalog-object (SSR)</h2>
|
||||||
|
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 6 }}>
|
||||||
|
Заполняет wall_type, energy_eff, ceiling_height_m, parking_*,
|
||||||
|
playground_*, scores из публичной страницы каталога DOM.РФ. По
|
||||||
|
умолчанию пропускает объекты, обновлённые сегодня.
|
||||||
|
</p>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
marginTop: 12,
|
||||||
|
display: "flex",
|
||||||
|
gap: 12,
|
||||||
|
alignItems: "flex-end",
|
||||||
|
flexWrap: "wrap",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<label>
|
||||||
|
<span style={labelStyle}>max_objects (пусто = 300)</span>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
min={1}
|
||||||
|
max={2000}
|
||||||
|
value={catalogMaxObjects}
|
||||||
|
onChange={(e) =>
|
||||||
|
setCatalogMaxObjects(
|
||||||
|
e.target.value === "" ? "" : Number(e.target.value),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
placeholder="300"
|
||||||
|
style={{ ...inputStyle, width: 140 }}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => catalogTrigger.mutate(false)}
|
||||||
|
disabled={!token || catalogTrigger.isPending}
|
||||||
|
style={primaryBtn}
|
||||||
|
title="Грузим только новые + не обновлённые сегодня"
|
||||||
|
>
|
||||||
|
{catalogTrigger.isPending && catalogTrigger.variables === false
|
||||||
|
? "Запускаю…"
|
||||||
|
: "Грузить (skip today)"}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
onClick={() => {
|
||||||
|
if (
|
||||||
|
confirm(
|
||||||
|
"Загрузить ВСЕ объекты последнего snapshot (force=true)? Игнорирует skip-today, нагрузка на DOM.РФ выше.",
|
||||||
|
)
|
||||||
|
) {
|
||||||
|
catalogTrigger.mutate(true);
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
disabled={!token || catalogTrigger.isPending}
|
||||||
|
style={{
|
||||||
|
...secondaryBtn,
|
||||||
|
color: "#b3261e",
|
||||||
|
borderColor: "#fecaca",
|
||||||
|
}}
|
||||||
|
title="Force: грузит все объекты независимо от catalog_scraped_at"
|
||||||
|
>
|
||||||
|
{catalogTrigger.isPending && catalogTrigger.variables === true
|
||||||
|
? "Запускаю…"
|
||||||
|
: "⚡ Загрузить ВСЕ (force)"}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{catalogTrigger.isError ? (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
marginTop: 12,
|
||||||
|
padding: 12,
|
||||||
|
background: "#fef2f2",
|
||||||
|
border: "1px solid #fecaca",
|
||||||
|
borderRadius: 6,
|
||||||
|
color: "#991b1b",
|
||||||
|
fontSize: 13,
|
||||||
|
wordBreak: "break-word",
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<strong>Ошибка:</strong> {String(catalogTrigger.error)}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
{catalogTrigger.isSuccess ? (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
marginTop: 12,
|
||||||
|
padding: 12,
|
||||||
|
background: "#f0fdf4",
|
||||||
|
border: "1px solid #bbf7d0",
|
||||||
|
borderRadius: 6,
|
||||||
|
color: "#065f46",
|
||||||
|
fontSize: 13,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<strong>✓ Catalog-object queued.</strong> task_id={" "}
|
||||||
|
<code
|
||||||
|
style={{
|
||||||
|
background: "#dcfce7",
|
||||||
|
padding: "2px 6px",
|
||||||
|
borderRadius: 3,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{catalogTrigger.data.task_id}
|
||||||
|
</code>{" "}
|
||||||
|
· регион {catalogTrigger.data.region_code} · force={" "}
|
||||||
|
{String(catalogTrigger.data.force)} · max_objects={" "}
|
||||||
|
{catalogTrigger.data.max_objects ?? "default"}
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
|
</section>
|
||||||
|
|
||||||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue