feat(scrapers/ui): backfill+full-load триггеры, source-фильтр runs, pacing-слайдер, coverage-таблица
Часть 1 — недостающие триггеры:
- CianBackfillSection: cian-backfill-history, cian-price-history (T8a), cian-full-load
- YandexBackfillSection: yandex-address-backfill (T10), yandex-full-load, yandex-newbuilding-sweep
- CianAutoLoginButton: POST /scrape/cian/auto-login в секцию Cookies
- HouseImvBackfillSection: house-imv-backfill (T7) в системный раздел
- GeocodeSection: geocode-missing-listings trigger + status polling
Часть 2 — source-фильтр в RunsTable:
- dropdown «Все / Avito / Cian / Yandex» + aria-label
- фильтр прокидывается в queryKey и query-param ?source=
Часть 3 — PacingControl:
- GET /api/v1/admin/scraper/pacing → PacingSection
- number-input + range slider (0–60с, шаг 0.5) + PUT /pacing/{source}
- env_default_s подсказка; graceful-degradation на ошибку/404
Часть 4 — DataQualitySection:
- GET /api/v1/admin/scraper/data-quality → таблица source × поле (% fill)
- <50% → красный акцент; th scope="col"/"row" a11y
- блок houses enrichment метрик; graceful-degradation на ошибку/404
This commit is contained in:
parent
a425f2f200
commit
00cd30b765
4 changed files with 1306 additions and 8 deletions
|
|
@ -1,7 +1,7 @@
|
|||
"use client";
|
||||
|
||||
import React, { useState } from "react";
|
||||
import { useMutation, useQuery } from "@tanstack/react-query";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
|
||||
import "@/components/trade-in/trade-in.css";
|
||||
import { Topbar } from "@/components/trade-in/Topbar";
|
||||
|
|
@ -19,6 +19,8 @@ import { SystemHealthSection } from "@/components/scrapers/ProxyHealthCard";
|
|||
import { RunsTable } from "@/components/scrapers/RunsTable";
|
||||
import { ScheduleControl } from "@/components/scrapers/ScheduleControl";
|
||||
import { ProviderProxySection } from "@/components/scrapers/ProviderProxySection";
|
||||
import { PacingSection } from "@/components/scrapers/PacingControl";
|
||||
import { DataQualitySection } from "@/components/scrapers/DataQualitySection";
|
||||
|
||||
// ── Types (provider-specific responses) ───────────────────────────────────
|
||||
|
||||
|
|
@ -128,6 +130,89 @@ interface YandexValuationTriggerResp {
|
|||
history_items_count: number;
|
||||
}
|
||||
|
||||
// ── Backfill / full-load response types ───────────────────────────────────
|
||||
|
||||
interface CianBackfillHistoryResp {
|
||||
ok: boolean;
|
||||
listings_total: number;
|
||||
listings_processed: number;
|
||||
listings_succeeded: number;
|
||||
listings_failed_fetch: number;
|
||||
listings_failed_save: number;
|
||||
price_changes_attempted: number;
|
||||
houses_total: number;
|
||||
houses_processed: number;
|
||||
houses_succeeded: number;
|
||||
houses_failed_fetch: number;
|
||||
houses_failed_save: number;
|
||||
valuations_total: number;
|
||||
valuations_processed: number;
|
||||
valuations_succeeded: number;
|
||||
valuations_failed: number;
|
||||
duration_sec: number;
|
||||
}
|
||||
|
||||
interface BackfillCountersResp {
|
||||
ok: boolean;
|
||||
checked: number;
|
||||
saved: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
duration_sec: number;
|
||||
}
|
||||
|
||||
interface HouseIMVBackfillResp {
|
||||
ok: boolean;
|
||||
checked: number;
|
||||
saved: number;
|
||||
skipped: number;
|
||||
errors: number;
|
||||
duration_sec: number;
|
||||
status_counts: Record<string, number>;
|
||||
}
|
||||
|
||||
interface GeocodeStatusResp {
|
||||
total_pending_listings: number;
|
||||
unique_addresses_pending: number;
|
||||
estimated_nominatim_minutes: number;
|
||||
cache_size_active: number;
|
||||
per_source: Array<{
|
||||
source: string;
|
||||
total: number;
|
||||
pending_geocode: number;
|
||||
has_coords: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface GeocodeBackfillResp {
|
||||
status: string;
|
||||
batch_size: number;
|
||||
dry_run: boolean;
|
||||
addresses_total?: number;
|
||||
addresses_processed?: number;
|
||||
addresses_geocoded?: number;
|
||||
addresses_failed?: number;
|
||||
listings_updated?: number;
|
||||
cache_hits?: number;
|
||||
cache_misses?: number;
|
||||
duration_sec?: number;
|
||||
note?: string;
|
||||
}
|
||||
|
||||
interface FullLoadStartResp {
|
||||
run_id: number;
|
||||
status: string;
|
||||
pages_per_anchor: number;
|
||||
detail_top_n: number;
|
||||
}
|
||||
|
||||
interface YandexNBSweepStartResp {
|
||||
run_id: number;
|
||||
status: string;
|
||||
limit: number;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
// ── Cookie helpers (Cian) ──────────────────────────────────────────────────
|
||||
|
||||
function convertCookiesFormat(input: string): Record<string, string> {
|
||||
|
|
@ -617,6 +702,123 @@ function AvitoAdvancedTriggers() {
|
|||
);
|
||||
}
|
||||
|
||||
// ── Cian Auto-Login ────────────────────────────────────────────────────────
|
||||
|
||||
interface AutoLoginResp {
|
||||
ok: boolean;
|
||||
userId?: number;
|
||||
cookieCount?: number;
|
||||
}
|
||||
|
||||
function CianAutoLoginButton() {
|
||||
const qc = useQueryClient();
|
||||
const [email, setEmail] = useState("");
|
||||
const [password, setPassword] = useState("");
|
||||
const [expanded, setExpanded] = useState(false);
|
||||
|
||||
const autoLoginMut = useMutation<AutoLoginResp, Error, { email: string; password: string } | null>({
|
||||
mutationFn: (body) =>
|
||||
apiFetch<AutoLoginResp>("/api/v1/admin/scrape/cian/auto-login", {
|
||||
method: "POST",
|
||||
body: body && (body.email || body.password) ? JSON.stringify(body) : undefined,
|
||||
}),
|
||||
onSuccess: () => {
|
||||
void qc.invalidateQueries({ queryKey: ["cian-test-auth"] });
|
||||
},
|
||||
});
|
||||
|
||||
return (
|
||||
<div style={{ marginTop: 16, borderTop: "1px solid var(--border-soft, #eef0f3)", paddingTop: 16 }}>
|
||||
<p style={{ fontSize: "0.85rem", color: "var(--fg-secondary, #5b6066)", marginBottom: 8 }}>
|
||||
<strong>Авто-логин</strong> — браузерный логин через camoufox (требует настроенных{" "}
|
||||
<code>CIAN_LOGIN_EMAIL</code>/<code>PASSWORD</code> в ENV или ввода ниже).
|
||||
</p>
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => setExpanded((v) => !v)}
|
||||
style={{
|
||||
background: "none",
|
||||
border: "none",
|
||||
padding: "4px 0",
|
||||
cursor: "pointer",
|
||||
fontSize: 13,
|
||||
fontWeight: 600,
|
||||
color: "var(--accent, #1d4ed8)",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
}}
|
||||
>
|
||||
<span>{expanded ? "▼" : "▶"}</span>
|
||||
Переопределить credentials (опционально)
|
||||
</button>
|
||||
{expanded && (
|
||||
<div className="scraper-form" style={{ marginTop: 8 }}>
|
||||
<label>
|
||||
Email
|
||||
<input
|
||||
type="email"
|
||||
value={email}
|
||||
onChange={(e) => setEmail(e.target.value)}
|
||||
placeholder="admin@example.com"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Password
|
||||
<input
|
||||
type="password"
|
||||
value={password}
|
||||
onChange={(e) => setPassword(e.target.value)}
|
||||
placeholder="из ENV по умолчанию"
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
)}
|
||||
<div style={{ marginTop: 10 }}>
|
||||
<button
|
||||
type="button"
|
||||
disabled={autoLoginMut.isPending}
|
||||
onClick={() =>
|
||||
autoLoginMut.mutate(
|
||||
email || password ? { email, password } : null,
|
||||
)
|
||||
}
|
||||
>
|
||||
{autoLoginMut.isPending ? "Логинимся…" : "Авто-логин Cian"}
|
||||
</button>
|
||||
</div>
|
||||
{autoLoginMut.isSuccess && (
|
||||
<div
|
||||
className="scraper-result"
|
||||
style={{
|
||||
marginTop: 8,
|
||||
background: autoLoginMut.data.ok
|
||||
? "var(--success-soft, #f0fdf4)"
|
||||
: "var(--danger-soft, #fef2f2)",
|
||||
color: autoLoginMut.data.ok
|
||||
? "var(--success, #0a7a3a)"
|
||||
: "var(--danger, #b3261e)",
|
||||
}}
|
||||
>
|
||||
{autoLoginMut.data.ok ? (
|
||||
<>
|
||||
<strong>Залогинились.</strong> userId={autoLoginMut.data.userId ?? "—"} ·
|
||||
cookieCount={autoLoginMut.data.cookieCount ?? "—"}
|
||||
</>
|
||||
) : (
|
||||
<strong>Авто-логин не удался.</strong>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
{autoLoginMut.isError && (
|
||||
<div className="scraper-result scraper-result--error" style={{ marginTop: 8 }}>
|
||||
<strong>Ошибка:</strong> {autoLoginMut.error.message}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Cian Cookies section ───────────────────────────────────────────────────
|
||||
|
||||
function CianCookiesSection() {
|
||||
|
|
@ -838,6 +1040,9 @@ function CianCookiesSection() {
|
|||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Auto-login (Variant B) */}
|
||||
<CianAutoLoginButton />
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
|
@ -1055,6 +1260,219 @@ function CianAdvancedTriggers() {
|
|||
Вызывается автоматически из /estimate flow (Stage 9). Admin trigger будет добавлен позже.
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<CianBackfillSection />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Cian Backfill section ─────────────────────────────────────────────────
|
||||
|
||||
function CianBackfillSection() {
|
||||
// ── cian-backfill-history ──────────────────────────────────────────────
|
||||
const [bfBatchSize, setBfBatchSize] = useState("50");
|
||||
const [bfDoListings, setBfDoListings] = useState(true);
|
||||
const [bfDoHouses, setBfDoHouses] = useState(true);
|
||||
const [bfDoValuations, setBfDoValuations] = useState(false);
|
||||
const [bfDryRun, setBfDryRun] = useState(false);
|
||||
|
||||
const backfillHistoryMut = useMutation<CianBackfillHistoryResp, Error>({
|
||||
mutationFn: () => {
|
||||
const qs = new URLSearchParams({
|
||||
batch_size: bfBatchSize,
|
||||
do_listings: String(bfDoListings),
|
||||
do_houses: String(bfDoHouses),
|
||||
do_valuations: String(bfDoValuations),
|
||||
dry_run: String(bfDryRun),
|
||||
});
|
||||
return apiFetch<CianBackfillHistoryResp>(
|
||||
`/api/v1/admin/scrape/cian-backfill-history?${qs.toString()}`,
|
||||
{ method: "POST" },
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
// ── cian-price-history ─────────────────────────────────────────────────
|
||||
const [phBatchSize, setPhBatchSize] = useState("50");
|
||||
const [phListingId, setPhListingId] = useState("");
|
||||
|
||||
const priceHistoryMut = useMutation<BackfillCountersResp, Error>({
|
||||
mutationFn: () =>
|
||||
apiFetch<BackfillCountersResp>("/api/v1/admin/scrape/cian-price-history", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
batch_size: parseInt(phBatchSize, 10),
|
||||
listing_id: phListingId !== "" ? parseInt(phListingId, 10) : null,
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
// ── cian-full-load ─────────────────────────────────────────────────────
|
||||
const [flDelay, setFlDelay] = useState("1.5");
|
||||
const [flConcurrency, setFlConcurrency] = useState("5");
|
||||
const [flPriceCap, setFlPriceCap] = useState("1400");
|
||||
const [flResumeRunId, setFlResumeRunId] = useState("");
|
||||
|
||||
const fullLoadMut = useMutation<FullLoadStartResp, Error>({
|
||||
mutationFn: () =>
|
||||
apiFetch<FullLoadStartResp>("/api/v1/admin/scrape/cian-full-load", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
price_cap_per_bucket: parseInt(flPriceCap, 10),
|
||||
request_delay_sec: parseFloat(flDelay),
|
||||
concurrency: parseInt(flConcurrency, 10),
|
||||
enrich_detail: false,
|
||||
detail_top_n: 0,
|
||||
resume_run_id: flResumeRunId !== "" ? parseInt(flResumeRunId, 10) : null,
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="scraper-section">
|
||||
<h3>5. Backfill — история цен (detail)</h3>
|
||||
<p className="scraper-hint">
|
||||
Дозаполняет offer_price_history / houses_price_dynamics для листингов и ЖК без истории.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
backfillHistoryMut.mutate();
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
batch_size
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={200}
|
||||
value={bfBatchSize}
|
||||
onChange={(e) => setBfBatchSize(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input type="checkbox" checked={bfDoListings} onChange={(e) => setBfDoListings(e.target.checked)} />
|
||||
listings
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input type="checkbox" checked={bfDoHouses} onChange={(e) => setBfDoHouses(e.target.checked)} />
|
||||
houses (ЖК)
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input type="checkbox" checked={bfDoValuations} onChange={(e) => setBfDoValuations(e.target.checked)} />
|
||||
valuations
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input type="checkbox" checked={bfDryRun} onChange={(e) => setBfDryRun(e.target.checked)} />
|
||||
dry_run
|
||||
</label>
|
||||
<button type="submit" disabled={backfillHistoryMut.isPending}>
|
||||
{backfillHistoryMut.isPending ? "Запускаем…" : "Запустить backfill-history"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={backfillHistoryMut} />
|
||||
</div>
|
||||
|
||||
<div className="scraper-section">
|
||||
<h3>6. Backfill — price-history (T8a)</h3>
|
||||
<p className="scraper-hint">
|
||||
T8a: Дозаполняет offer_price_history через detail-страницы (curl_cffi). Идемпотентен.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
priceHistoryMut.mutate();
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
batch_size
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={500}
|
||||
value={phBatchSize}
|
||||
onChange={(e) => setPhBatchSize(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
listing_id (опционально, debug)
|
||||
<input
|
||||
type="number"
|
||||
value={phListingId}
|
||||
onChange={(e) => setPhListingId(e.target.value)}
|
||||
placeholder="оставьте пустым"
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={priceHistoryMut.isPending}>
|
||||
{priceHistoryMut.isPending ? "Запускаем…" : "Запустить price-history backfill"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={priceHistoryMut} />
|
||||
</div>
|
||||
|
||||
<div className="scraper-section">
|
||||
<h3>7. Full Load (полный обход без anchor'ов)</h3>
|
||||
<p className="scraper-hint">
|
||||
Exhaustive Cian ЕКБ: адаптивное деление по цене, пагинация каждого бакета. Запускается в background.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
fullLoadMut.mutate();
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
price_cap_per_bucket
|
||||
<input
|
||||
type="number"
|
||||
min={200}
|
||||
max={1500}
|
||||
value={flPriceCap}
|
||||
onChange={(e) => setFlPriceCap(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
request_delay_sec
|
||||
<input
|
||||
type="number"
|
||||
min={0.5}
|
||||
max={15}
|
||||
step={0.5}
|
||||
value={flDelay}
|
||||
onChange={(e) => setFlDelay(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
concurrency
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={10}
|
||||
value={flConcurrency}
|
||||
onChange={(e) => setFlConcurrency(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
resume_run_id (опционально)
|
||||
<input
|
||||
type="number"
|
||||
value={flResumeRunId}
|
||||
onChange={(e) => setFlResumeRunId(e.target.value)}
|
||||
placeholder="оставьте пустым для нового прогона"
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={fullLoadMut.isPending}>
|
||||
{fullLoadMut.isPending ? "Запускаем…" : "Запустить Cian Full Load"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={fullLoadMut} />
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -1296,6 +1714,221 @@ function YandexAdvancedTriggers() {
|
|||
</form>
|
||||
<ResultPanel mut={valMut} />
|
||||
</div>
|
||||
|
||||
<YandexBackfillSection />
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Yandex Backfill section ────────────────────────────────────────────────
|
||||
|
||||
function YandexBackfillSection() {
|
||||
// ── yandex-address-backfill ────────────────────────────────────────────
|
||||
const [abLimit, setAbLimit] = useState("200");
|
||||
const [abDelay, setAbDelay] = useState("3.0");
|
||||
|
||||
const addressBackfillMut = useMutation<BackfillCountersResp, Error>({
|
||||
mutationFn: () =>
|
||||
apiFetch<BackfillCountersResp>("/api/v1/admin/scrape/yandex-address-backfill", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
limit: parseInt(abLimit, 10),
|
||||
request_delay_sec: parseFloat(abDelay),
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
// ── yandex-full-load ──────────────────────────────────────────────────
|
||||
const [flDelay, setFlDelay] = useState("2.0");
|
||||
const [flConcurrency, setFlConcurrency] = useState("4");
|
||||
const [flPriceCap, setFlPriceCap] = useState("500");
|
||||
const [flResumeRunId, setFlResumeRunId] = useState("");
|
||||
|
||||
const fullLoadMut = useMutation<FullLoadStartResp, Error>({
|
||||
mutationFn: () =>
|
||||
apiFetch<FullLoadStartResp>("/api/v1/admin/scrape/yandex-full-load", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
price_cap_per_bucket: parseInt(flPriceCap, 10),
|
||||
request_delay_sec: parseFloat(flDelay),
|
||||
concurrency: parseInt(flConcurrency, 10),
|
||||
resume_run_id: flResumeRunId !== "" ? parseInt(flResumeRunId, 10) : null,
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
// ── yandex-newbuilding-sweep ───────────────────────────────────────────
|
||||
const [nbLimit, setNbLimit] = useState("5");
|
||||
const [nbDelay, setNbDelay] = useState("8.0");
|
||||
const [nbCity, setNbCity] = useState("ekaterinburg");
|
||||
const [nbForce, setNbForce] = useState(false);
|
||||
|
||||
const nbSweepMut = useMutation<YandexNBSweepStartResp, Error>({
|
||||
mutationFn: () =>
|
||||
apiFetch<YandexNBSweepStartResp>("/api/v1/admin/scrape/yandex-newbuilding-sweep", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
limit: parseInt(nbLimit, 10),
|
||||
request_delay_sec: parseFloat(nbDelay),
|
||||
force: nbForce,
|
||||
city: nbCity,
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
return (
|
||||
<>
|
||||
<div className="scraper-section">
|
||||
<h3>5. Backfill — address (T10)</h3>
|
||||
<p className="scraper-hint">
|
||||
Дозаполняет listings.address для Yandex-листингов без номера дома (detail-страница → полный адрес).
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
addressBackfillMut.mutate();
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
limit
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={2000}
|
||||
value={abLimit}
|
||||
onChange={(e) => setAbLimit(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
request_delay_sec
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={15}
|
||||
step={0.5}
|
||||
value={abDelay}
|
||||
onChange={(e) => setAbDelay(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={addressBackfillMut.isPending}>
|
||||
{addressBackfillMut.isPending ? "Запускаем…" : "Запустить address backfill"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={addressBackfillMut} />
|
||||
</div>
|
||||
|
||||
<div className="scraper-section">
|
||||
<h3>6. Full Load (полный обход без anchor'ов)</h3>
|
||||
<p className="scraper-hint">
|
||||
Exhaustive Yandex ЕКБ: адаптивное деление по цене, пагинация каждого бакета. Background.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
fullLoadMut.mutate();
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
price_cap_per_bucket
|
||||
<input
|
||||
type="number"
|
||||
min={100}
|
||||
max={575}
|
||||
value={flPriceCap}
|
||||
onChange={(e) => setFlPriceCap(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
request_delay_sec
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={15}
|
||||
step={0.5}
|
||||
value={flDelay}
|
||||
onChange={(e) => setFlDelay(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
concurrency
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={8}
|
||||
value={flConcurrency}
|
||||
onChange={(e) => setFlConcurrency(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
resume_run_id (опционально)
|
||||
<input
|
||||
type="number"
|
||||
value={flResumeRunId}
|
||||
onChange={(e) => setFlResumeRunId(e.target.value)}
|
||||
placeholder="оставьте пустым"
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={fullLoadMut.isPending}>
|
||||
{fullLoadMut.isPending ? "Запускаем…" : "Запустить Yandex Full Load"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={fullLoadMut} />
|
||||
</div>
|
||||
|
||||
<div className="scraper-section">
|
||||
<h3>7. Newbuilding enrichment sweep</h3>
|
||||
<p className="scraper-hint">
|
||||
Обогащение pending ЖК-домов: resolve slug → fetch_jk → UPSERT yandex_jk_enrichment.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
nbSweepMut.mutate();
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
limit
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={200}
|
||||
value={nbLimit}
|
||||
onChange={(e) => setNbLimit(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
request_delay_sec
|
||||
<input
|
||||
type="number"
|
||||
min={3}
|
||||
max={30}
|
||||
step={0.5}
|
||||
value={nbDelay}
|
||||
onChange={(e) => setNbDelay(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
city
|
||||
<input
|
||||
value={nbCity}
|
||||
onChange={(e) => setNbCity(e.target.value)}
|
||||
placeholder="ekaterinburg"
|
||||
/>
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input type="checkbox" checked={nbForce} onChange={(e) => setNbForce(e.target.checked)} />
|
||||
force (перезаписать уже обогащённые)
|
||||
</label>
|
||||
<button type="submit" disabled={nbSweepMut.isPending}>
|
||||
{nbSweepMut.isPending ? "Запускаем…" : "Запустить NB sweep"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={nbSweepMut} />
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -1378,6 +2011,188 @@ function PerSourceDelayEditor({ source, delayMin, delayMax }: PerSourceDelayEdit
|
|||
);
|
||||
}
|
||||
|
||||
// ── Cross-source Backfill & Sync (System tab) ─────────────────────────────
|
||||
|
||||
function HouseImvBackfillSection() {
|
||||
const [batchSize, setBatchSize] = useState("50");
|
||||
const [delay, setDelay] = useState("5.0");
|
||||
const [onlyStatus, setOnlyStatus] = useState("pending");
|
||||
const [houseId, setHouseId] = useState("");
|
||||
|
||||
const imvMut = useMutation<HouseIMVBackfillResp, Error>({
|
||||
mutationFn: () =>
|
||||
apiFetch<HouseIMVBackfillResp>("/api/v1/admin/scrape/house-imv-backfill", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({
|
||||
batch_size: parseInt(batchSize, 10),
|
||||
request_delay_sec: parseFloat(delay),
|
||||
only_status: onlyStatus,
|
||||
house_id: houseId !== "" ? parseInt(houseId, 10) : null,
|
||||
}),
|
||||
}),
|
||||
});
|
||||
|
||||
return (
|
||||
<section className="scraper-section">
|
||||
<h2>House IMV Backfill (T7)</h2>
|
||||
<p className="scraper-hint">
|
||||
Batch оценка домов через Avito IMV API. Итерирует дома по{" "}
|
||||
<code>imv_status</code>, сохраняет в{" "}
|
||||
<code>house_imv_evaluations</code> / <code>house_placement_history</code> /{" "}
|
||||
<code>house_suggestions</code>. Не снижать delay ниже 3s.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
imvMut.mutate();
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
batch_size
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={500}
|
||||
value={batchSize}
|
||||
onChange={(e) => setBatchSize(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
request_delay_sec
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={30}
|
||||
step={0.5}
|
||||
value={delay}
|
||||
onChange={(e) => setDelay(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
only_status
|
||||
<select value={onlyStatus} onChange={(e) => setOnlyStatus(e.target.value)}>
|
||||
<option value="pending">pending (обработать новые)</option>
|
||||
<option value="transient_error">transient_error (retry failed)</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
house_id (опционально, debug)
|
||||
<input
|
||||
type="number"
|
||||
value={houseId}
|
||||
onChange={(e) => setHouseId(e.target.value)}
|
||||
placeholder="оставьте пустым"
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={imvMut.isPending} style={{ gridColumn: "1 / -1" }}>
|
||||
{imvMut.isPending ? "Запускаем IMV backfill…" : "Запустить House IMV backfill"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={imvMut} />
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
function GeocodeSection() {
|
||||
const [batchSize, setGcBatchSize] = useState("200");
|
||||
const [dryRun, setGcDryRun] = useState(false);
|
||||
const [background, setGcBackground] = useState(false);
|
||||
|
||||
const statusQ = useQuery<GeocodeStatusResp>({
|
||||
queryKey: ["geocode-missing-status"],
|
||||
queryFn: () =>
|
||||
apiFetch<GeocodeStatusResp>("/api/v1/admin/scrape/geocode-missing-listings/status"),
|
||||
staleTime: 30_000,
|
||||
refetchInterval: 60_000,
|
||||
retry: 1,
|
||||
});
|
||||
|
||||
const geocodeMut = useMutation<GeocodeBackfillResp, Error>({
|
||||
mutationFn: () => {
|
||||
const qs = new URLSearchParams({
|
||||
batch_size: batchSize,
|
||||
dry_run: String(dryRun),
|
||||
background: String(background),
|
||||
});
|
||||
return apiFetch<GeocodeBackfillResp>(
|
||||
`/api/v1/admin/scrape/geocode-missing-listings?${qs.toString()}`,
|
||||
{ method: "POST" },
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
return (
|
||||
<section className="scraper-section">
|
||||
<h2>Geocode missing listings</h2>
|
||||
<p className="scraper-hint">
|
||||
Дозаполняет координаты листингов через Nominatim (address-dedup).
|
||||
Безопасно повторять. Background=true → возвращает немедленно, статус — в логах.
|
||||
</p>
|
||||
|
||||
{/* Status block */}
|
||||
{statusQ.isError && (
|
||||
<p className="scraper-result scraper-result--error" style={{ marginBottom: 8 }}>
|
||||
Ошибка загрузки статуса: {statusQ.error.message}
|
||||
</p>
|
||||
)}
|
||||
{statusQ.isSuccess && (
|
||||
<div className="schedule-status" style={{ marginBottom: 12 }}>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Pending listings:</span>
|
||||
<span>{statusQ.data.total_pending_listings.toLocaleString("ru-RU")}</span>
|
||||
</div>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Уникальных адресов:</span>
|
||||
<span>{statusQ.data.unique_addresses_pending.toLocaleString("ru-RU")}</span>
|
||||
</div>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">~Минут (Nominatim):</span>
|
||||
<span>{statusQ.data.estimated_nominatim_minutes}</span>
|
||||
</div>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Geocode cache (active):</span>
|
||||
<span>{statusQ.data.cache_size_active.toLocaleString("ru-RU")}</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
geocodeMut.mutate();
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
batch_size
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={2000}
|
||||
value={batchSize}
|
||||
onChange={(e) => setGcBatchSize(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input type="checkbox" checked={dryRun} onChange={(e) => setGcDryRun(e.target.checked)} />
|
||||
dry_run (показать без записи)
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input type="checkbox" checked={background} onChange={(e) => setGcBackground(e.target.checked)} />
|
||||
background (не ждать результат)
|
||||
</label>
|
||||
<button type="submit" disabled={geocodeMut.isPending} style={{ gridColumn: "1 / -1" }}>
|
||||
{geocodeMut.isPending ? "Геокодируем…" : "Запустить geocode backfill"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={geocodeMut} />
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Provider configs ───────────────────────────────────────────────────────
|
||||
|
||||
interface ProviderConfig {
|
||||
|
|
@ -1502,9 +2317,19 @@ export default function ScrapersUnifiedPage() {
|
|||
{/* Section 1: Система */}
|
||||
<SystemHealthSection />
|
||||
|
||||
{/* Section 2: Глобально */}
|
||||
{/* Section 2: Pacing */}
|
||||
<PacingSection />
|
||||
|
||||
{/* Section 3: Глобально */}
|
||||
<GlobalSection />
|
||||
|
||||
{/* Section 4: Backfill & Sync (cross-source) */}
|
||||
<HouseImvBackfillSection />
|
||||
<GeocodeSection />
|
||||
|
||||
{/* Section 5: Качество данных */}
|
||||
<DataQualitySection />
|
||||
|
||||
{/* Tabs: Avito / Cian / Yandex */}
|
||||
<div style={{ marginTop: 24 }}>
|
||||
{/* Tab bar */}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,224 @@
|
|||
"use client";
|
||||
|
||||
import React from "react";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { apiFetch } from "@/lib/api";
|
||||
|
||||
// ── Types (per contract) ───────────────────────────────────────────────────
|
||||
|
||||
interface SourceQuality {
|
||||
source: string;
|
||||
active_count: number;
|
||||
fields: Record<string, number>;
|
||||
}
|
||||
|
||||
interface HousesEnrichment {
|
||||
total: number;
|
||||
validated_pct: number;
|
||||
rating_pct: number;
|
||||
house_type_pct: number;
|
||||
reviews_count: number;
|
||||
}
|
||||
|
||||
interface DataQualityResp {
|
||||
sources: SourceQuality[];
|
||||
houses: HousesEnrichment;
|
||||
}
|
||||
|
||||
// ── Hook ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function useDataQuality() {
|
||||
return useQuery<DataQualityResp>({
|
||||
queryKey: ["scraper-data-quality"],
|
||||
queryFn: () =>
|
||||
apiFetch<DataQualityResp>("/api/v1/admin/scraper/data-quality"),
|
||||
staleTime: 60_000,
|
||||
refetchInterval: 120_000,
|
||||
retry: 1,
|
||||
});
|
||||
}
|
||||
|
||||
// ── Fill cell ─────────────────────────────────────────────────────────────
|
||||
|
||||
function FillCell({ pct }: { pct: number }) {
|
||||
const isLow = pct < 50;
|
||||
return (
|
||||
<td
|
||||
style={{
|
||||
textAlign: "right",
|
||||
fontSize: "0.8rem",
|
||||
fontWeight: isLow ? 600 : 400,
|
||||
color: isLow ? "var(--danger, #b3261e)" : "var(--fg-primary, #1f2937)",
|
||||
background: isLow ? "var(--danger-soft, #fef2f2)" : undefined,
|
||||
}}
|
||||
>
|
||||
{pct.toFixed(1)}%
|
||||
</td>
|
||||
);
|
||||
}
|
||||
|
||||
// ── DataQualitySection (exported) ─────────────────────────────────────────
|
||||
|
||||
export function DataQualitySection() {
|
||||
const qualityQ = useDataQuality();
|
||||
|
||||
// Collect all unique field names across all sources
|
||||
const allFields: string[] = qualityQ.isSuccess
|
||||
? [
|
||||
...new Set<string>(
|
||||
qualityQ.data.sources.flatMap((s) => Object.keys(s.fields)),
|
||||
),
|
||||
].sort()
|
||||
: [];
|
||||
|
||||
return (
|
||||
<section className="scraper-section">
|
||||
<h2>Качество данных / Coverage</h2>
|
||||
<p className="scraper-hint">
|
||||
Процент заполнения ключевых полей по активным листингам и обогащению
|
||||
домов. Красный — менее 50%. Обновление каждые 2 мин.
|
||||
</p>
|
||||
|
||||
{/* Graceful degradation: API not yet deployed */}
|
||||
{qualityQ.isError && (
|
||||
<p
|
||||
className="scraper-result scraper-result--error"
|
||||
style={{ marginBottom: 0 }}
|
||||
>
|
||||
coverage API недоступен
|
||||
{" — "}
|
||||
<span style={{ color: "var(--fg-secondary, #5b6066)" }}>
|
||||
{qualityQ.error.message}
|
||||
</span>
|
||||
</p>
|
||||
)}
|
||||
|
||||
{qualityQ.isPending && (
|
||||
<p className="scraper-hint">Загрузка данных coverage…</p>
|
||||
)}
|
||||
|
||||
{qualityQ.isSuccess && (
|
||||
<>
|
||||
{/* Sources × fields table */}
|
||||
{qualityQ.data.sources.length > 0 && allFields.length > 0 && (
|
||||
<div style={{ overflowX: "auto", marginBottom: 20 }}>
|
||||
<table className="runs-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col">Source</th>
|
||||
<th scope="col" style={{ textAlign: "right" }}>
|
||||
Активных
|
||||
</th>
|
||||
{allFields.map((f) => (
|
||||
<th
|
||||
key={f}
|
||||
scope="col"
|
||||
style={{ textAlign: "right", whiteSpace: "nowrap" }}
|
||||
>
|
||||
{f}
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{qualityQ.data.sources.map((row) => (
|
||||
<tr key={row.source}>
|
||||
<th scope="row">
|
||||
<code style={{ fontWeight: 600 }}>{row.source}</code>
|
||||
</th>
|
||||
<td style={{ textAlign: "right", fontSize: "0.85rem" }}>
|
||||
{row.active_count.toLocaleString("ru-RU")}
|
||||
</td>
|
||||
{allFields.map((f) => (
|
||||
<FillCell
|
||||
key={f}
|
||||
pct={row.fields[f] !== undefined ? row.fields[f] : 0}
|
||||
/>
|
||||
))}
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{qualityQ.data.sources.length === 0 && (
|
||||
<p className="scraper-hint">Нет данных по источникам.</p>
|
||||
)}
|
||||
|
||||
{/* Houses enrichment metrics */}
|
||||
<div>
|
||||
<p
|
||||
style={{
|
||||
fontSize: "0.85rem",
|
||||
fontWeight: 600,
|
||||
marginBottom: 8,
|
||||
}}
|
||||
>
|
||||
Обогащение домов
|
||||
</p>
|
||||
<div className="schedule-status">
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Всего домов:</span>
|
||||
<span>
|
||||
{qualityQ.data.houses.total.toLocaleString("ru-RU")}
|
||||
</span>
|
||||
</div>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Валидировано:</span>
|
||||
<span
|
||||
style={{
|
||||
color:
|
||||
qualityQ.data.houses.validated_pct < 50
|
||||
? "var(--danger, #b3261e)"
|
||||
: undefined,
|
||||
fontWeight:
|
||||
qualityQ.data.houses.validated_pct < 50 ? 600 : 400,
|
||||
}}
|
||||
>
|
||||
{qualityQ.data.houses.validated_pct.toFixed(1)}%
|
||||
</span>
|
||||
</div>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">С рейтингом:</span>
|
||||
<span
|
||||
style={{
|
||||
color:
|
||||
qualityQ.data.houses.rating_pct < 50
|
||||
? "var(--danger, #b3261e)"
|
||||
: undefined,
|
||||
fontWeight:
|
||||
qualityQ.data.houses.rating_pct < 50 ? 600 : 400,
|
||||
}}
|
||||
>
|
||||
{qualityQ.data.houses.rating_pct.toFixed(1)}%
|
||||
</span>
|
||||
</div>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">С типом дома:</span>
|
||||
<span
|
||||
style={{
|
||||
color:
|
||||
qualityQ.data.houses.house_type_pct < 50
|
||||
? "var(--danger, #b3261e)"
|
||||
: undefined,
|
||||
fontWeight:
|
||||
qualityQ.data.houses.house_type_pct < 50 ? 600 : 400,
|
||||
}}
|
||||
>
|
||||
{qualityQ.data.houses.house_type_pct.toFixed(1)}%
|
||||
</span>
|
||||
</div>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Отзывов (sum):</span>
|
||||
<span>
|
||||
{qualityQ.data.houses.reviews_count.toLocaleString("ru-RU")}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
198
tradein-mvp/frontend/src/components/scrapers/PacingControl.tsx
Normal file
198
tradein-mvp/frontend/src/components/scrapers/PacingControl.tsx
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
"use client";
|
||||
|
||||
import React, { useState, useEffect } from "react";
|
||||
import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { apiFetch } from "@/lib/api";
|
||||
|
||||
// ── Types (per contract) ───────────────────────────────────────────────────
|
||||
|
||||
interface PacingProvider {
|
||||
source: "avito" | "cian" | "yandex" | "generic";
|
||||
interval_s: number;
|
||||
env_default_s: number;
|
||||
}
|
||||
|
||||
interface PacingResp {
|
||||
providers: PacingProvider[];
|
||||
}
|
||||
|
||||
interface PacingUpdateResp {
|
||||
ok: boolean;
|
||||
source: string;
|
||||
interval_s: number;
|
||||
}
|
||||
|
||||
// ── Hooks ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function usePacing() {
|
||||
return useQuery<PacingResp>({
|
||||
queryKey: ["scraper-pacing"],
|
||||
queryFn: () => apiFetch<PacingResp>("/api/v1/admin/scraper/pacing"),
|
||||
staleTime: 30_000,
|
||||
retry: 1,
|
||||
});
|
||||
}
|
||||
|
||||
function useUpdatePacing(source: string) {
|
||||
const qc = useQueryClient();
|
||||
return useMutation<PacingUpdateResp, Error, { interval_s: number }>({
|
||||
mutationFn: ({ interval_s }) =>
|
||||
apiFetch<PacingUpdateResp>(
|
||||
`/api/v1/admin/scraper/pacing/${encodeURIComponent(source)}`,
|
||||
{
|
||||
method: "PUT",
|
||||
body: JSON.stringify({ interval_s }),
|
||||
},
|
||||
),
|
||||
onSuccess: () => {
|
||||
void qc.invalidateQueries({ queryKey: ["scraper-pacing"] });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ── Per-provider row ───────────────────────────────────────────────────────
|
||||
|
||||
interface PacingRowProps {
|
||||
provider: PacingProvider;
|
||||
}
|
||||
|
||||
function PacingRow({ provider }: PacingRowProps) {
|
||||
const [value, setValue] = useState<string>(String(provider.interval_s));
|
||||
const updateMut = useUpdatePacing(provider.source);
|
||||
|
||||
// Sync when upstream data refreshes
|
||||
useEffect(() => {
|
||||
setValue(String(provider.interval_s));
|
||||
}, [provider.interval_s]);
|
||||
|
||||
const numValue = parseFloat(value);
|
||||
const isValid = !isNaN(numValue) && numValue >= 0 && numValue <= 60;
|
||||
|
||||
return (
|
||||
<tr>
|
||||
<td>
|
||||
<code style={{ fontWeight: 600 }}>{provider.source}</code>
|
||||
</td>
|
||||
<td>
|
||||
<div style={{ display: "flex", alignItems: "center", gap: 8, flexWrap: "wrap" }}>
|
||||
<input
|
||||
type="number"
|
||||
min={0}
|
||||
max={60}
|
||||
step={0.5}
|
||||
value={value}
|
||||
onChange={(e) => setValue(e.target.value)}
|
||||
aria-label={`Pacing interval for ${provider.source} (seconds)`}
|
||||
style={{
|
||||
width: 72,
|
||||
padding: "3px 6px",
|
||||
fontSize: "0.85rem",
|
||||
border: "1px solid var(--border-card, #e6e8ec)",
|
||||
borderRadius: 6,
|
||||
}}
|
||||
/>
|
||||
<input
|
||||
type="range"
|
||||
min={0}
|
||||
max={60}
|
||||
step={0.5}
|
||||
value={isValid ? numValue : 0}
|
||||
onChange={(e) => setValue(e.target.value)}
|
||||
aria-label={`Pacing slider for ${provider.source}`}
|
||||
style={{ width: 120 }}
|
||||
/>
|
||||
<span style={{ fontSize: "0.8rem", color: "var(--fg-tertiary, #73767e)" }}>
|
||||
с (по умолч. {provider.env_default_s} с — сбросится при рестарте)
|
||||
</span>
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<button
|
||||
type="button"
|
||||
disabled={updateMut.isPending || !isValid}
|
||||
onClick={() => updateMut.mutate({ interval_s: numValue })}
|
||||
style={{ fontSize: "0.8rem", padding: "3px 10px" }}
|
||||
>
|
||||
{updateMut.isPending ? "…" : "Применить"}
|
||||
</button>
|
||||
{updateMut.isSuccess && updateMut.data.ok && (
|
||||
<span
|
||||
style={{ marginLeft: 8, fontSize: "0.75rem", color: "var(--success, #0a7a3a)" }}
|
||||
>
|
||||
{updateMut.data.interval_s} с — ок
|
||||
</span>
|
||||
)}
|
||||
{updateMut.isSuccess && !updateMut.data.ok && (
|
||||
<span
|
||||
style={{ marginLeft: 8, fontSize: "0.75rem", color: "var(--danger, #b3261e)" }}
|
||||
>
|
||||
не применено
|
||||
</span>
|
||||
)}
|
||||
{updateMut.isError && (
|
||||
<span
|
||||
style={{ marginLeft: 8, fontSize: "0.75rem", color: "var(--danger, #b3261e)" }}
|
||||
>
|
||||
{updateMut.error.message}
|
||||
</span>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
}
|
||||
|
||||
// ── PacingSection (exported) ───────────────────────────────────────────────
|
||||
|
||||
export function PacingSection() {
|
||||
const pacingQ = usePacing();
|
||||
|
||||
return (
|
||||
<section className="scraper-section">
|
||||
<h2>Pacing (интервалы запросов)</h2>
|
||||
<p className="scraper-hint">
|
||||
Минимальный интервал между запросами скраппера для каждого провайдера.
|
||||
0 = без искусственной паузы. Изменение вступает в силу немедленно;
|
||||
сбрасывается при рестарте контейнера (постоянное значение — через ENV).
|
||||
</p>
|
||||
|
||||
{/* Graceful degradation: API not yet deployed */}
|
||||
{pacingQ.isError && (
|
||||
<p
|
||||
className="scraper-result scraper-result--error"
|
||||
style={{ marginBottom: 0 }}
|
||||
>
|
||||
pacing API недоступен
|
||||
{" — "}
|
||||
<span style={{ color: "var(--fg-secondary, #5b6066)" }}>
|
||||
{pacingQ.error.message}
|
||||
</span>
|
||||
</p>
|
||||
)}
|
||||
|
||||
{pacingQ.isPending && (
|
||||
<p className="scraper-hint">Загрузка pacing…</p>
|
||||
)}
|
||||
|
||||
{pacingQ.isSuccess && pacingQ.data.providers.length === 0 && (
|
||||
<p className="scraper-hint">Нет данных pacing от API.</p>
|
||||
)}
|
||||
|
||||
{pacingQ.isSuccess && pacingQ.data.providers.length > 0 && (
|
||||
<table className="runs-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th scope="col">Провайдер</th>
|
||||
<th scope="col">Интервал (0–60 с)</th>
|
||||
<th scope="col">Действие</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{pacingQ.data.providers.map((p) => (
|
||||
<PacingRow key={p.source} provider={p} />
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
|
@ -37,15 +37,35 @@ interface RunsListResp {
|
|||
const RUN_STATUS_ALL = ["", "running", "done", "failed", "cancelled", "zombie", "banned"] as const;
|
||||
type RunStatusFilter = (typeof RUN_STATUS_ALL)[number];
|
||||
|
||||
// "" means "all sources"; otherwise a specific source prefix (avito / cian / yandex)
|
||||
const RUN_SOURCE_FILTERS = ["", "avito", "cian", "yandex"] as const;
|
||||
type RunSourceFilter = (typeof RUN_SOURCE_FILTERS)[number];
|
||||
|
||||
const SOURCE_FILTER_LABELS: Record<RunSourceFilter, string> = {
|
||||
"": "Все",
|
||||
avito: "Avito",
|
||||
cian: "Cian",
|
||||
yandex: "Yandex",
|
||||
};
|
||||
|
||||
function useScraperRuns(
|
||||
source: ScraperSource,
|
||||
status: RunStatusFilter,
|
||||
sourceFilter: RunSourceFilter,
|
||||
limit = 20,
|
||||
) {
|
||||
return useQuery<RunsListResp>({
|
||||
queryKey: ["scrape-runs", source, status, limit],
|
||||
queryKey: ["scrape-runs", source, status, sourceFilter, limit],
|
||||
queryFn: () => {
|
||||
const qs = new URLSearchParams({ source, limit: String(limit) });
|
||||
const qs = new URLSearchParams({ limit: String(limit) });
|
||||
// When a specific sourceFilter is chosen, ignore the tab-level source
|
||||
// and pass it verbatim as the ?source= param
|
||||
if (sourceFilter) {
|
||||
qs.set("source", sourceFilter);
|
||||
} else {
|
||||
// fallback: filter by the current tab provider
|
||||
qs.set("source", source);
|
||||
}
|
||||
if (status) qs.set("status", status);
|
||||
return apiFetch<RunsListResp>(
|
||||
`/api/v1/admin/scrape/runs?${qs.toString()}`,
|
||||
|
|
@ -154,17 +174,17 @@ interface RunsTableProps {
|
|||
}
|
||||
|
||||
export function RunsTable({ source }: RunsTableProps) {
|
||||
const [statusFilter, setStatusFilter] =
|
||||
useState<RunStatusFilter>("");
|
||||
const [statusFilter, setStatusFilter] = useState<RunStatusFilter>("");
|
||||
const [sourceFilter, setSourceFilter] = useState<RunSourceFilter>("");
|
||||
const qc = useQueryClient();
|
||||
const runsQ = useScraperRuns(source, statusFilter);
|
||||
const runsQ = useScraperRuns(source, statusFilter, sourceFilter);
|
||||
const cancelMut = useCancelCitySweep(source);
|
||||
|
||||
function handleCancel(runId: number) {
|
||||
cancelMut.mutate(runId, {
|
||||
onSuccess: () => {
|
||||
void qc.invalidateQueries({
|
||||
queryKey: ["scrape-runs", source, statusFilter],
|
||||
queryKey: ["scrape-runs", source, statusFilter, sourceFilter],
|
||||
});
|
||||
},
|
||||
});
|
||||
|
|
@ -177,6 +197,37 @@ export function RunsTable({ source }: RunsTableProps) {
|
|||
Последние 20 прогонов. Автообновление каждые 8 сек.
|
||||
</p>
|
||||
|
||||
{/* Source filter */}
|
||||
<div style={{ marginBottom: 8, display: "flex", gap: 8, alignItems: "center", flexWrap: "wrap" }}>
|
||||
<label
|
||||
htmlFor="runs-source-filter"
|
||||
style={{ fontSize: "0.8rem", color: "var(--fg-secondary, #5b6066)", whiteSpace: "nowrap" }}
|
||||
>
|
||||
Источник:
|
||||
</label>
|
||||
<select
|
||||
id="runs-source-filter"
|
||||
value={sourceFilter}
|
||||
onChange={(e) => setSourceFilter(e.target.value as RunSourceFilter)}
|
||||
style={{
|
||||
fontSize: "0.8rem",
|
||||
padding: "3px 8px",
|
||||
borderRadius: 6,
|
||||
border: "1px solid var(--border-card, #e6e8ec)",
|
||||
background: "var(--bg-card, #fff)",
|
||||
color: "var(--fg-primary, #1f2937)",
|
||||
cursor: "pointer",
|
||||
}}
|
||||
aria-label="Фильтр по источнику"
|
||||
>
|
||||
{RUN_SOURCE_FILTERS.map((sf) => (
|
||||
<option key={sf || "_all"} value={sf}>
|
||||
{SOURCE_FILTER_LABELS[sf]}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
{/* Status filter */}
|
||||
<div style={{ marginBottom: 12, display: "flex", gap: 8, flexWrap: "wrap" }}>
|
||||
{RUN_STATUS_ALL.map((s) => (
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue