feat(tradein): /scrapers/avito — 5-я секция City sweep (auto + cancel + runs list) (#478)

2026-05-23 14:40:11 +00:00 · 2026-05-23 14:40:11 +00:00 · dd24bd8136
commit dd24bd8136
parent 2914ff2ef2
2 changed files with 409 additions and 1 deletions
--- a/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx
+++ b/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx
@ -1,7 +1,7 @@
 "use client";

 import { useState } from "react";
-import { useMutation } from "@tanstack/react-query";
+import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";

 import "@/components/trade-in/trade-in.css";
 import { Topbar } from "@/components/trade-in/Topbar";
@ -115,6 +115,94 @@ function useScrapeImv() {
  });
 }

+// ── City Sweep types + hooks ───────────────────────────────────────────────
+
+interface CitySweepInput {
+  pages_per_anchor: number;
+  detail_top_n: number;
+  request_delay_sec: number;
+  enrich_houses: boolean;
+  radius_m: number;
+}
+
+interface CitySweepStartResp {
+  run_id: number;
+  status: string;
+  pages_per_anchor: number;
+  detail_top_n: number;
+}
+
+interface CitySweepCounters {
+  anchors_total?: number;
+  anchors_done?: number;
+  lots_fetched?: number;
+  lots_inserted?: number;
+  lots_updated?: number;
+  unique_houses?: number;
+  houses_enriched?: number;
+  houses_failed?: number;
+  detail_attempted?: number;
+  detail_enriched?: number;
+  detail_failed?: number;
+  errors_count?: number;
+}
+
+interface ScrapeRunRow {
+  run_id: number;
+  source: string;
+  status: string;
+  params: Record<string, unknown> | null;
+  counters: CitySweepCounters | null;
+  error: string | null;
+  started_at: string | null;
+  finished_at: string | null;
+  heartbeat_at: string | null;
+}
+
+function useStartCitySweep() {
+  const qc = useQueryClient();
+  return useMutation<CitySweepStartResp, Error, CitySweepInput>({
+    mutationFn: (input) =>
+      apiFetch<CitySweepStartResp>("/api/v1/admin/scrape/avito-city-sweep", {
+        method: "POST",
+        body: JSON.stringify(input),
+      }),
+    onSuccess: () => {
+      qc.invalidateQueries({ queryKey: ["city-sweep-runs"] });
+    },
+  });
+}
+
+function useCitySweepRuns() {
+  return useQuery<ScrapeRunRow[]>({
+    queryKey: ["city-sweep-runs"],
+    queryFn: () =>
+      apiFetch<ScrapeRunRow[]>(
+        "/api/v1/admin/scrape/avito-city-sweep/runs?limit=10",
+      ),
+    refetchInterval: 5_000,
+    staleTime: 0,
+  });
+}
+
+function useCancelCitySweep() {
+  const qc = useQueryClient();
+  return useMutation<
+    { ok: boolean; run_id: number; cancelled: boolean },
+    Error,
+    number
+  >({
+    mutationFn: (run_id) =>
+      apiFetch(
+        `/api/v1/admin/scrape/avito-city-sweep/${run_id}/cancel`,
+        { method: "POST" },
+      ),
+    onSuccess: () => {
+      qc.invalidateQueries({ queryKey: ["city-sweep-runs"] });
+    },
+  });
+}
+
 // ── Result panel ───────────────────────────────────────────────────────────

 interface ResultPanelProps<TData> {
@ -179,6 +267,16 @@ export default function AvitoScraperPage() {
  const [imvLoggia, setImvLoggia] = useState(false);
  const imvMut = useScrapeImv();

+  // City sweep state
+  const [sweepPages, setSweepPages] = useState("3");
+  const [sweepTopN, setSweepTopN] = useState("20");
+  const [sweepDelay, setSweepDelay] = useState("7");
+  const [sweepRadius, setSweepRadius] = useState("1500");
+  const [sweepEnrichHouses, setSweepEnrichHouses] = useState(true);
+  const sweepMut = useStartCitySweep();
+  const sweepRuns = useCitySweepRuns();
+  const cancelMut = useCancelCitySweep();
+
  return (
    <>
      <Topbar active="avito" />
@ -411,7 +509,211 @@ export default function AvitoScraperPage() {
          </form>
          <ResultPanel mut={imvMut} />
        </section>
+
+        {/* 5. City sweep ЕКБ */}
+        <section className="scraper-section scraper-section--highlight">
+          <h2>5. Полный обход ЕКБ (auto sweep)</h2>
+          <p className="scraper-hint">
+            Iterate 5 anchors × N pages → save listings → enrich houses + detail top-N.
+            Запускается в background (~15-30 мин для defaults). Можно отменить через
+            «Отменить» в списке ниже. Cron автоматически запускает каждый день в случайное
+            время 02:00-05:00 UTC.
+          </p>
+          <form
+            className="scraper-form"
+            onSubmit={(e) => {
+              e.preventDefault();
+              sweepMut.mutate({
+                pages_per_anchor: parseInt(sweepPages, 10),
+                detail_top_n: parseInt(sweepTopN, 10),
+                request_delay_sec: parseFloat(sweepDelay),
+                enrich_houses: sweepEnrichHouses,
+                radius_m: parseInt(sweepRadius, 10),
+              });
+            }}
+          >
+            <label>
+              Страниц на anchor
+              <input
+                type="number"
+                min={1}
+                max={10}
+                value={sweepPages}
+                onChange={(e) => setSweepPages(e.target.value)}
+              />
+            </label>
+            <label>
+              Detail top-N
+              <input
+                type="number"
+                min={0}
+                max={30}
+                value={sweepTopN}
+                onChange={(e) => setSweepTopN(e.target.value)}
+              />
+            </label>
+            <label>
+              Delay (сек)
+              <input
+                type="number"
+                step={0.5}
+                min={3}
+                max={15}
+                value={sweepDelay}
+                onChange={(e) => setSweepDelay(e.target.value)}
+              />
+            </label>
+            <label>
+              Радиус (м)
+              <input
+                type="number"
+                min={500}
+                max={5000}
+                step={100}
+                value={sweepRadius}
+                onChange={(e) => setSweepRadius(e.target.value)}
+              />
+            </label>
+            <label className="checkbox">
+              <input
+                type="checkbox"
+                checked={sweepEnrichHouses}
+                onChange={(e) => setSweepEnrichHouses(e.target.checked)}
+              />
+              Enrich houses
+            </label>
+            <button type="submit" disabled={sweepMut.isPending}>
+              {sweepMut.isPending ? "Запускаем…" : "Запустить sweep"}
+            </button>
+          </form>
+
+          {sweepMut.error && (
+            <div className="scraper-result scraper-result--error">
+              <strong>Ошибка:</strong> {sweepMut.error.message}
+            </div>
+          )}
+          {sweepMut.isSuccess && sweepMut.data && (
+            <div className="scraper-result">
+              <pre>{JSON.stringify(sweepMut.data, null, 2)}</pre>
+            </div>
+          )}
+
+          {/* Recent runs list */}
+          <div className="city-sweep-runs">
+            <h3>Recent runs (auto-refresh 5s)</h3>
+            {sweepRuns.isPending && <p className="scraper-hint">Загрузка…</p>}
+            {sweepRuns.error && (
+              <p className="scraper-result scraper-result--error">
+                Ошибка загрузки runs: {sweepRuns.error.message}
+              </p>
+            )}
+            {sweepRuns.data && sweepRuns.data.length === 0 && (
+              <p className="scraper-hint">Нет запусков пока.</p>
+            )}
+            {sweepRuns.data && sweepRuns.data.length > 0 && (
+              <table className="runs-table">
+                <thead>
+                  <tr>
+                    <th>#</th>
+                    <th>Статус</th>
+                    <th>Старт</th>
+                    <th>Heartbeat</th>
+                    <th>Anchors</th>
+                    <th>Lots</th>
+                    <th>Houses</th>
+                    <th>Detail</th>
+                    <th>Errors</th>
+                    <th>Действие</th>
+                  </tr>
+                </thead>
+                <tbody>
+                  {sweepRuns.data.map((r) => (
+                    <tr key={r.run_id} className={`run-row run-row--${r.status}`}>
+                      <td>{r.run_id}</td>
+                      <td>
+                        <span className={`run-badge run-badge--${r.status}`}>
+                          {translateStatus(r.status)}
+                        </span>
+                      </td>
+                      <td>{formatTime(r.started_at)}</td>
+                      <td>{formatTime(r.heartbeat_at)}</td>
+                      <td>
+                        {r.counters?.anchors_done ?? 0}/{r.counters?.anchors_total ?? 0}
+                      </td>
+                      <td>
+                        {r.counters?.lots_fetched ?? 0}{" "}
+                        <span className="run-muted">
+                          (+{r.counters?.lots_inserted ?? 0})
+                        </span>
+                      </td>
+                      <td>
+                        {r.counters?.houses_enriched ?? 0}/{r.counters?.unique_houses ?? 0}
+                      </td>
+                      <td>
+                        {r.counters?.detail_enriched ?? 0}/{r.counters?.detail_attempted ?? 0}
+                      </td>
+                      <td>{r.counters?.errors_count ?? 0}</td>
+                      <td>
+                        {r.status === "running" && (
+                          <button
+                            type="button"
+                            className="cancel-btn"
+                            disabled={
+                              cancelMut.isPending &&
+                              cancelMut.variables === r.run_id
+                            }
+                            onClick={() => cancelMut.mutate(r.run_id)}
+                          >
+                            Отменить
+                          </button>
+                        )}
+                      </td>
+                    </tr>
+                  ))}
+                </tbody>
+              </table>
+            )}
+          </div>
+        </section>
      </main>
    </>
  );
 }
+
+// ── Helpers ────────────────────────────────────────────────────────────────
+
+function translateStatus(s: string): string {
+  switch (s) {
+    case "running":
+      return "выполняется";
+    case "done":
+      return "готово";
+    case "failed":
+      return "ошибка";
+    case "cancelled":
+      return "отменено";
+    case "zombie":
+      return "зомби";
+    case "skipped":
+      return "пропущено";
+    case "banned":
+      return "блок";
+    default:
+      return s;
+  }
+}
+
+function formatTime(iso: string | null): string {
+  if (!iso) return "—";
+  try {
+    return new Date(iso).toLocaleString("ru-RU", {
+      day: "2-digit",
+      month: "2-digit",
+      hour: "2-digit",
+      minute: "2-digit",
+      second: "2-digit",
+    });
+  } catch {
+    return iso;
+  }
+}
--- a/tradein-mvp/frontend/src/components/trade-in/trade-in.css
+++ b/tradein-mvp/frontend/src/components/trade-in/trade-in.css
@ -1372,3 +1372,109 @@
  white-space: pre-wrap;
  word-break: break-all;
 }
+
+/* ── Stage 4d: city sweep section ── */
+
+.scraper-section--highlight {
+  border-color: var(--accent, #1d4ed8);
+  background: var(--accent-soft, #dbeafe);
+}
+
+.scraper-section--highlight h2 {
+  color: var(--accent, #1d4ed8);
+}
+
+.city-sweep-runs {
+  margin-top: 20px;
+  padding-top: 16px;
+  border-top: 1px solid var(--border-card, #e6e8ec);
+}
+
+.city-sweep-runs h3 {
+  font-size: 14px;
+  font-weight: 600;
+  margin: 0 0 12px;
+  color: var(--fg-primary, #111111);
+}
+
+.runs-table {
+  width: 100%;
+  border-collapse: collapse;
+  font-size: 12px;
+  background: #ffffff;
+  border-radius: 6px;
+  overflow: hidden;
+}
+
+.runs-table thead {
+  background: var(--bg-card-alt, #fafbfc);
+}
+
+.runs-table th {
+  padding: 8px 10px;
+  text-align: left;
+  font-weight: 600;
+  font-size: 11px;
+  color: var(--fg-secondary, #5b6066);
+  text-transform: uppercase;
+  letter-spacing: 0.04em;
+  border-bottom: 1px solid var(--border-card, #e6e8ec);
+}
+
+.runs-table td {
+  padding: 8px 10px;
+  border-bottom: 1px solid var(--border-soft, #eef0f3);
+  font-variant-numeric: tabular-nums;
+}
+
+.runs-table .run-muted {
+  color: var(--fg-secondary, #5b6066);
+  font-size: 11px;
+}
+
+.run-badge {
+  padding: 3px 8px;
+  border-radius: 4px;
+  font-size: 11px;
+  font-weight: 500;
+}
+
+.run-badge--running {
+  background: var(--accent-soft, #dbeafe);
+  color: var(--accent, #1d4ed8);
+}
+
+.run-badge--done {
+  background: var(--success-soft, #dcfce7);
+  color: var(--success, #0a7a3a);
+}
+
+.run-badge--failed,
+.run-badge--zombie {
+  background: var(--danger-soft, #fee2e2);
+  color: var(--danger, #b3261e);
+}
+
+.run-badge--cancelled {
+  background: var(--warn-soft, #fef3c7);
+  color: var(--warn, #9a6700);
+}
+
+.cancel-btn {
+  padding: 4px 10px;
+  background: var(--danger, #b3261e);
+  color: #ffffff;
+  border: none;
+  border-radius: 4px;
+  font-size: 11px;
+  cursor: pointer;
+}
+
+.cancel-btn:hover:not(:disabled) {
+  background: #8b1d18;
+}
+
+.cancel-btn:disabled {
+  background: var(--border-strong, #d1d5db);
+  cursor: not-allowed;
+}