feat(22d): admin UI buttons (skip/force) + reduce catalog jitter 800→300ms #352
2 changed files with 143 additions and 2 deletions
|
|
@ -190,7 +190,7 @@ async def fetch_catalog_object_html(session: BrowserSession, obj_id: int) -> str
|
|||
|
||||
for attempt in range(5):
|
||||
async with session._sem:
|
||||
await jitter_sleep(800, 1500)
|
||||
await jitter_sleep(300, 700)
|
||||
try:
|
||||
session._request_count += 1
|
||||
result = await session._page.evaluate(_FETCH_HTML_JS, {"url": url})
|
||||
|
|
@ -412,7 +412,7 @@ async def scrape_catalog_objects(
|
|||
) -> dict[str, int]:
|
||||
"""Scrape списка объектов через один BrowserSession.
|
||||
|
||||
Запускает один BrowserSession на весь batch; jitter_sleep (800–1500 мс)
|
||||
Запускает один BrowserSession на весь batch; jitter_sleep (300–700 мс)
|
||||
встроен в fetch_catalog_object_html для защиты от rate-limit.
|
||||
|
||||
Returns:
|
||||
|
|
|
|||
|
|
@ -26,6 +26,13 @@ interface TriggerResp {
|
|||
developers: string[] | null;
|
||||
}
|
||||
|
||||
interface CatalogTriggerResp {
|
||||
task_id: string;
|
||||
region_code: number;
|
||||
max_objects: number | null;
|
||||
force: boolean;
|
||||
}
|
||||
|
||||
interface QueueTask {
|
||||
worker: string;
|
||||
task_id: string;
|
||||
|
|
@ -78,6 +85,7 @@ export default function ScrapeAdminPage() {
|
|||
const [extras, setExtras] = useState(true);
|
||||
const [downloadPhotos, setDownloadPhotos] = useState(false);
|
||||
const [forceRun, setForceRun] = useState(false);
|
||||
const [catalogMaxObjects, setCatalogMaxObjects] = useState<number | "">("");
|
||||
const [showFailures, setShowFailures] = useState(false);
|
||||
const [failuresRunId, setFailuresRunId] = useState<number | "">("");
|
||||
const [logsRunId, setLogsRunId] = useState<number | "">("");
|
||||
|
|
@ -121,6 +129,27 @@ export default function ScrapeAdminPage() {
|
|||
},
|
||||
});
|
||||
|
||||
const catalogTrigger = useMutation({
|
||||
mutationFn: (force: boolean) =>
|
||||
apiFetch<CatalogTriggerResp>("/api/v1/admin/scrape/kn-catalog-objects", {
|
||||
method: "POST",
|
||||
headers: { "X-Admin-Token": token },
|
||||
body: JSON.stringify({
|
||||
region_code: region,
|
||||
max_objects: catalogMaxObjects === "" ? null : catalogMaxObjects,
|
||||
force,
|
||||
}),
|
||||
}),
|
||||
onSuccess: (data) => {
|
||||
console.log("[scrape] catalog queued:", data);
|
||||
queryClient.invalidateQueries({ queryKey: ["scrape-runs"] });
|
||||
queryClient.invalidateQueries({ queryKey: ["scrape-queue"] });
|
||||
},
|
||||
onError: (err) => {
|
||||
console.error("[scrape] catalog failed:", err);
|
||||
},
|
||||
});
|
||||
|
||||
const releaseLock = useMutation({
|
||||
mutationFn: () =>
|
||||
apiFetch<{
|
||||
|
|
@ -458,6 +487,118 @@ export default function ScrapeAdminPage() {
|
|||
) : null}
|
||||
</section>
|
||||
|
||||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||
<h2 style={{ margin: 0, fontSize: 16 }}>📦 Catalog-object (SSR)</h2>
|
||||
<p style={{ color: "#5b6066", fontSize: 13, marginTop: 6 }}>
|
||||
Заполняет wall_type, energy_eff, ceiling_height_m, parking_*,
|
||||
playground_*, scores из публичной страницы каталога DOM.РФ. По
|
||||
умолчанию пропускает объекты, обновлённые сегодня.
|
||||
</p>
|
||||
<div
|
||||
style={{
|
||||
marginTop: 12,
|
||||
display: "flex",
|
||||
gap: 12,
|
||||
alignItems: "flex-end",
|
||||
flexWrap: "wrap",
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
<span style={labelStyle}>max_objects (пусто = 300)</span>
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={2000}
|
||||
value={catalogMaxObjects}
|
||||
onChange={(e) =>
|
||||
setCatalogMaxObjects(
|
||||
e.target.value === "" ? "" : Number(e.target.value),
|
||||
)
|
||||
}
|
||||
placeholder="300"
|
||||
style={{ ...inputStyle, width: 140 }}
|
||||
/>
|
||||
</label>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => catalogTrigger.mutate(false)}
|
||||
disabled={!token || catalogTrigger.isPending}
|
||||
style={primaryBtn}
|
||||
title="Грузим только новые + не обновлённые сегодня"
|
||||
>
|
||||
{catalogTrigger.isPending && catalogTrigger.variables === false
|
||||
? "Запускаю…"
|
||||
: "Грузить (skip today)"}
|
||||
</button>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => {
|
||||
if (
|
||||
confirm(
|
||||
"Загрузить ВСЕ объекты последнего snapshot (force=true)? Игнорирует skip-today, нагрузка на DOM.РФ выше.",
|
||||
)
|
||||
) {
|
||||
catalogTrigger.mutate(true);
|
||||
}
|
||||
}}
|
||||
disabled={!token || catalogTrigger.isPending}
|
||||
style={{
|
||||
...secondaryBtn,
|
||||
color: "#b3261e",
|
||||
borderColor: "#fecaca",
|
||||
}}
|
||||
title="Force: грузит все объекты независимо от catalog_scraped_at"
|
||||
>
|
||||
{catalogTrigger.isPending && catalogTrigger.variables === true
|
||||
? "Запускаю…"
|
||||
: "⚡ Загрузить ВСЕ (force)"}
|
||||
</button>
|
||||
</div>
|
||||
{catalogTrigger.isError ? (
|
||||
<div
|
||||
style={{
|
||||
marginTop: 12,
|
||||
padding: 12,
|
||||
background: "#fef2f2",
|
||||
border: "1px solid #fecaca",
|
||||
borderRadius: 6,
|
||||
color: "#991b1b",
|
||||
fontSize: 13,
|
||||
wordBreak: "break-word",
|
||||
}}
|
||||
>
|
||||
<strong>Ошибка:</strong> {String(catalogTrigger.error)}
|
||||
</div>
|
||||
) : null}
|
||||
{catalogTrigger.isSuccess ? (
|
||||
<div
|
||||
style={{
|
||||
marginTop: 12,
|
||||
padding: 12,
|
||||
background: "#f0fdf4",
|
||||
border: "1px solid #bbf7d0",
|
||||
borderRadius: 6,
|
||||
color: "#065f46",
|
||||
fontSize: 13,
|
||||
}}
|
||||
>
|
||||
<strong>✓ Catalog-object queued.</strong> task_id={" "}
|
||||
<code
|
||||
style={{
|
||||
background: "#dcfce7",
|
||||
padding: "2px 6px",
|
||||
borderRadius: 3,
|
||||
}}
|
||||
>
|
||||
{catalogTrigger.data.task_id}
|
||||
</code>{" "}
|
||||
· регион {catalogTrigger.data.region_code} · force={" "}
|
||||
{String(catalogTrigger.data.force)} · max_objects={" "}
|
||||
{catalogTrigger.data.max_objects ?? "default"}
|
||||
</div>
|
||||
) : null}
|
||||
</section>
|
||||
|
||||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||
<div
|
||||
style={{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue