feat(22d): admin UI buttons (skip/force) + reduce catalog jitter 800→300ms #352

Merged
lekss361 merged 1 commit from feat/22d-catalog-ui-buttons-fast-jitter into main 2026-05-17 23:38:22 +00:00
2 changed files with 143 additions and 2 deletions

View file

@ -190,7 +190,7 @@ async def fetch_catalog_object_html(session: BrowserSession, obj_id: int) -> str
for attempt in range(5):
async with session._sem:
await jitter_sleep(800, 1500)
await jitter_sleep(300, 700)
try:
session._request_count += 1
result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url})
@ -412,7 +412,7 @@ async def scrape_catalog_objects(
) -> dict[str, int]:
"""Scrape списка объектов через один BrowserSession.
Запускает один BrowserSession на весь batch; jitter_sleep (8001500 мс)
Запускает один BrowserSession на весь batch; jitter_sleep (300700 мс)
встроен в fetch_catalog_object_html для защиты от rate-limit.
Returns:

View file

@ -26,6 +26,13 @@ interface TriggerResp {
developers: string[] | null;
}
interface CatalogTriggerResp {
task_id: string;
region_code: number;
max_objects: number | null;
force: boolean;
}
interface QueueTask {
worker: string;
task_id: string;
@ -78,6 +85,7 @@ export default function ScrapeAdminPage() {
const [extras, setExtras] = useState(true);
const [downloadPhotos, setDownloadPhotos] = useState(false);
const [forceRun, setForceRun] = useState(false);
const [catalogMaxObjects, setCatalogMaxObjects] = useState<number | "">("");
const [showFailures, setShowFailures] = useState(false);
const [failuresRunId, setFailuresRunId] = useState<number | "">("");
const [logsRunId, setLogsRunId] = useState<number | "">("");
@ -121,6 +129,27 @@ export default function ScrapeAdminPage() {
},
});
const catalogTrigger = useMutation({
mutationFn: (force: boolean) =>
apiFetch<CatalogTriggerResp>("/api/v1/admin/scrape/kn-catalog-objects", {
method: "POST",
headers: { "X-Admin-Token": token },
body: JSON.stringify({
region_code: region,
max_objects: catalogMaxObjects === "" ? null : catalogMaxObjects,
force,
}),
}),
onSuccess: (data) => {
console.log("[scrape] catalog queued:", data);
queryClient.invalidateQueries({ queryKey: ["scrape-runs"] });
queryClient.invalidateQueries({ queryKey: ["scrape-queue"] });
},
onError: (err) => {
console.error("[scrape] catalog failed:", err);
},
});
const releaseLock = useMutation({
mutationFn: () =>
apiFetch<{
@ -458,6 +487,118 @@ export default function ScrapeAdminPage() {
) : null}
</section>
<section style={{ ...cardStyle, marginTop: 16 }}>
<h2 style={{ margin: 0, fontSize: 16 }}>📦 Catalog-object (SSR)</h2>
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 6 }}>
Заполняет wall_type, energy_eff, ceiling_height_m, parking_*,
playground_*, scores из публичной страницы каталога DOM.РФ. По
умолчанию пропускает объекты, обновлённые сегодня.
</p>
<div
style={{
marginTop: 12,
display: "flex",
gap: 12,
alignItems: "flex-end",
flexWrap: "wrap",
}}
>
<label>
<span style={labelStyle}>max_objects (пусто = 300)</span>
<input
type="number"
min={1}
max={2000}
value={catalogMaxObjects}
onChange={(e) =>
setCatalogMaxObjects(
e.target.value === "" ? "" : Number(e.target.value),
)
}
placeholder="300"
style={{ ...inputStyle, width: 140 }}
/>
</label>
<button
type="button"
onClick={() => catalogTrigger.mutate(false)}
disabled={!token || catalogTrigger.isPending}
style={primaryBtn}
title="Грузим только новые + не обновлённые сегодня"
>
{catalogTrigger.isPending && catalogTrigger.variables === false
? "Запускаю…"
: "Грузить (skip today)"}
</button>
<button
type="button"
onClick={() => {
if (
confirm(
"Загрузить ВСЕ объекты последнего snapshot (force=true)? Игнорирует skip-today, нагрузка на DOM.РФ выше.",
)
) {
catalogTrigger.mutate(true);
}
}}
disabled={!token || catalogTrigger.isPending}
style={{
...secondaryBtn,
color: "#b3261e",
borderColor: "#fecaca",
}}
title="Force: грузит все объекты независимо от catalog_scraped_at"
>
{catalogTrigger.isPending && catalogTrigger.variables === true
? "Запускаю…"
: "⚡ Загрузить ВСЕ (force)"}
</button>
</div>
{catalogTrigger.isError ? (
<div
style={{
marginTop: 12,
padding: 12,
background: "#fef2f2",
border: "1px solid #fecaca",
borderRadius: 6,
color: "#991b1b",
fontSize: 13,
wordBreak: "break-word",
}}
>
<strong>Ошибка:</strong> {String(catalogTrigger.error)}
</div>
) : null}
{catalogTrigger.isSuccess ? (
<div
style={{
marginTop: 12,
padding: 12,
background: "#f0fdf4",
border: "1px solid #bbf7d0",
borderRadius: 6,
color: "#065f46",
fontSize: 13,
}}
>
<strong> Catalog-object queued.</strong> task_id={" "}
<code
style={{
background: "#dcfce7",
padding: "2px 6px",
borderRadius: 3,
}}
>
{catalogTrigger.data.task_id}
</code>{" "}
· регион {catalogTrigger.data.region_code} · force={" "}
{String(catalogTrigger.data.force)} · max_objects={" "}
{catalogTrigger.data.max_objects ?? "default"}
</div>
) : null}
</section>
<section style={{ ...cardStyle, marginTop: 16 }}>
<div
style={{