From 1d82a56a6ac8d8cf95174da25c4ca7f6c1380e03 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 17:32:28 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20/scrapers/avito=20=E2=80=94=20?= =?UTF-8?q?5-=D1=8F=20=D1=81=D0=B5=D0=BA=D1=86=D0=B8=D1=8F=20City=20sweep?= =?UTF-8?q?=20(auto=20+=20cancel=20+=20runs=20list)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - City sweep form: pages_per_anchor (1-10), detail_top_n (0-30), request_delay_sec (3-15), radius_m, enrich_houses checkbox -> POST /admin/scrape/avito-city-sweep - 'Запустить sweep' button -> returns run_id (background task) - Recent runs table (auto-poll 5s): status badge, started/heartbeat, counters (anchors_done/total, lots, houses_enriched/unique, detail_enriched/attempted, errors) - 'Отменить' button per running run -> POST /scrape/avito-city-sweep/{run_id}/cancel - Status badges с цветами (running/done/failed/cancelled/zombie) - translateStatus helper (RU labels), formatTime helper (ru-RU locale) - CSS: .scraper-section--highlight (синий border для city-sweep section), .city-sweep-runs, .runs-table с tabular-nums, .run-badge variants, .cancel-btn Backend контракт: 3 endpoints из parallel PR (avito-city-sweep start/runs/cancel). TS strict + next build pass. --- .../frontend/src/app/scrapers/avito/page.tsx | 304 +++++++++++++++++- .../src/components/trade-in/trade-in.css | 106 ++++++ 2 files changed, 409 insertions(+), 1 deletion(-) diff --git a/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx b/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx index 23263f2f..7c961ecf 100644 --- a/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx +++ b/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx @@ -1,7 +1,7 @@ "use client"; import { useState } from "react"; -import { useMutation } from "@tanstack/react-query"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import "@/components/trade-in/trade-in.css"; import { Topbar } from "@/components/trade-in/Topbar"; @@ -115,6 +115,94 @@ function useScrapeImv() { }); } +// ── City Sweep types + hooks ─────────────────────────────────────────────── + +interface CitySweepInput { + pages_per_anchor: number; + detail_top_n: number; + request_delay_sec: number; + enrich_houses: boolean; + radius_m: number; +} + +interface CitySweepStartResp { + run_id: number; + status: string; + pages_per_anchor: number; + detail_top_n: number; +} + +interface CitySweepCounters { + anchors_total?: number; + anchors_done?: number; + lots_fetched?: number; + lots_inserted?: number; + lots_updated?: number; + unique_houses?: number; + houses_enriched?: number; + houses_failed?: number; + detail_attempted?: number; + detail_enriched?: number; + detail_failed?: number; + errors_count?: number; +} + +interface ScrapeRunRow { + run_id: number; + source: string; + status: string; + params: Record | null; + counters: CitySweepCounters | null; + error: string | null; + started_at: string | null; + finished_at: string | null; + heartbeat_at: string | null; +} + +function useStartCitySweep() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (input) => + apiFetch("/api/v1/admin/scrape/avito-city-sweep", { + method: "POST", + body: JSON.stringify(input), + }), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["city-sweep-runs"] }); + }, + }); +} + +function useCitySweepRuns() { + return useQuery({ + queryKey: ["city-sweep-runs"], + queryFn: () => + apiFetch( + "/api/v1/admin/scrape/avito-city-sweep/runs?limit=10", + ), + refetchInterval: 5_000, + staleTime: 0, + }); +} + +function useCancelCitySweep() { + const qc = useQueryClient(); + return useMutation< + { ok: boolean; run_id: number; cancelled: boolean }, + Error, + number + >({ + mutationFn: (run_id) => + apiFetch( + `/api/v1/admin/scrape/avito-city-sweep/${run_id}/cancel`, + { method: "POST" }, + ), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["city-sweep-runs"] }); + }, + }); +} + // ── Result panel ─────────────────────────────────────────────────────────── interface ResultPanelProps { @@ -179,6 +267,16 @@ export default function AvitoScraperPage() { const [imvLoggia, setImvLoggia] = useState(false); const imvMut = useScrapeImv(); + // City sweep state + const [sweepPages, setSweepPages] = useState("3"); + const [sweepTopN, setSweepTopN] = useState("20"); + const [sweepDelay, setSweepDelay] = useState("7"); + const [sweepRadius, setSweepRadius] = useState("1500"); + const [sweepEnrichHouses, setSweepEnrichHouses] = useState(true); + const sweepMut = useStartCitySweep(); + const sweepRuns = useCitySweepRuns(); + const cancelMut = useCancelCitySweep(); + return ( <> @@ -411,7 +509,211 @@ export default function AvitoScraperPage() { + + {/* 5. City sweep ЕКБ */} +
+

5. Полный обход ЕКБ (auto sweep)

+

+ Iterate 5 anchors × N pages → save listings → enrich houses + detail top-N. + Запускается в background (~15-30 мин для defaults). Можно отменить через + «Отменить» в списке ниже. Cron автоматически запускает каждый день в случайное + время 02:00-05:00 UTC. +

+
{ + e.preventDefault(); + sweepMut.mutate({ + pages_per_anchor: parseInt(sweepPages, 10), + detail_top_n: parseInt(sweepTopN, 10), + request_delay_sec: parseFloat(sweepDelay), + enrich_houses: sweepEnrichHouses, + radius_m: parseInt(sweepRadius, 10), + }); + }} + > + + + + + + +
+ + {sweepMut.error && ( +
+ Ошибка: {sweepMut.error.message} +
+ )} + {sweepMut.isSuccess && sweepMut.data && ( +
+ 
{JSON.stringify(sweepMut.data, null, 2)}
+
+ )} + + {/* Recent runs list */} +
+

Recent runs (auto-refresh 5s)

+ {sweepRuns.isPending &&

Загрузка…

} + {sweepRuns.error && ( +

+ Ошибка загрузки runs: {sweepRuns.error.message} +

+ )} + {sweepRuns.data && sweepRuns.data.length === 0 && ( +

Нет запусков пока.

+ )} + {sweepRuns.data && sweepRuns.data.length > 0 && ( + + + + + + + + + + + + + + + + + {sweepRuns.data.map((r) => ( + + + + + + + + + + + + + ))} + +
#СтатусСтартHeartbeatAnchorsLotsHousesDetailErrorsДействие
{r.run_id} + + {translateStatus(r.status)} + + {formatTime(r.started_at)}{formatTime(r.heartbeat_at)} + {r.counters?.anchors_done ?? 0}/{r.counters?.anchors_total ?? 0} + + {r.counters?.lots_fetched ?? 0}{" "} + + (+{r.counters?.lots_inserted ?? 0}) + + + {r.counters?.houses_enriched ?? 0}/{r.counters?.unique_houses ?? 0} + + {r.counters?.detail_enriched ?? 0}/{r.counters?.detail_attempted ?? 0} + {r.counters?.errors_count ?? 0} + {r.status === "running" && ( + + )} +
+ )} +
+
); } + +// ── Helpers ──────────────────────────────────────────────────────────────── + +function translateStatus(s: string): string { + switch (s) { + case "running": + return "выполняется"; + case "done": + return "готово"; + case "failed": + return "ошибка"; + case "cancelled": + return "отменено"; + case "zombie": + return "зомби"; + case "skipped": + return "пропущено"; + case "banned": + return "блок"; + default: + return s; + } +} + +function formatTime(iso: string | null): string { + if (!iso) return "—"; + try { + return new Date(iso).toLocaleString("ru-RU", { + day: "2-digit", + month: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); + } catch { + return iso; + } +} diff --git a/tradein-mvp/frontend/src/components/trade-in/trade-in.css b/tradein-mvp/frontend/src/components/trade-in/trade-in.css index c0c6514d..d905993e 100644 --- a/tradein-mvp/frontend/src/components/trade-in/trade-in.css +++ b/tradein-mvp/frontend/src/components/trade-in/trade-in.css @@ -1372,3 +1372,109 @@ white-space: pre-wrap; word-break: break-all; } + +/* ── Stage 4d: city sweep section ── */ + +.scraper-section--highlight { + border-color: var(--accent, #1d4ed8); + background: var(--accent-soft, #dbeafe); +} + +.scraper-section--highlight h2 { + color: var(--accent, #1d4ed8); +} + +.city-sweep-runs { + margin-top: 20px; + padding-top: 16px; + border-top: 1px solid var(--border-card, #e6e8ec); +} + +.city-sweep-runs h3 { + font-size: 14px; + font-weight: 600; + margin: 0 0 12px; + color: var(--fg-primary, #111111); +} + +.runs-table { + width: 100%; + border-collapse: collapse; + font-size: 12px; + background: #ffffff; + border-radius: 6px; + overflow: hidden; +} + +.runs-table thead { + background: var(--bg-card-alt, #fafbfc); +} + +.runs-table th { + padding: 8px 10px; + text-align: left; + font-weight: 600; + font-size: 11px; + color: var(--fg-secondary, #5b6066); + text-transform: uppercase; + letter-spacing: 0.04em; + border-bottom: 1px solid var(--border-card, #e6e8ec); +} + +.runs-table td { + padding: 8px 10px; + border-bottom: 1px solid var(--border-soft, #eef0f3); + font-variant-numeric: tabular-nums; +} + +.runs-table .run-muted { + color: var(--fg-secondary, #5b6066); + font-size: 11px; +} + +.run-badge { + padding: 3px 8px; + border-radius: 4px; + font-size: 11px; + font-weight: 500; +} + +.run-badge--running { + background: var(--accent-soft, #dbeafe); + color: var(--accent, #1d4ed8); +} + +.run-badge--done { + background: var(--success-soft, #dcfce7); + color: var(--success, #0a7a3a); +} + +.run-badge--failed, +.run-badge--zombie { + background: var(--danger-soft, #fee2e2); + color: var(--danger, #b3261e); +} + +.run-badge--cancelled { + background: var(--warn-soft, #fef3c7); + color: var(--warn, #9a6700); +} + +.cancel-btn { + padding: 4px 10px; + background: var(--danger, #b3261e); + color: #ffffff; + border: none; + border-radius: 4px; + font-size: 11px; + cursor: pointer; +} + +.cancel-btn:hover:not(:disabled) { + background: #8b1d18; +} + +.cancel-btn:disabled { + background: var(--border-strong, #d1d5db); + cursor: not-allowed; +}