From 0b07d75fb6e6ec5a2074f386e8bb504a50c5072d Mon Sep 17 00:00:00 2001 From: bot-frontend Date: Sun, 31 May 2026 12:21:02 +0300 Subject: [PATCH] feat(tradein): unify scraper admin pages with shared ScraperPage component MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each provider page (avito/cian/yandex) now has identical structure via a shared component: trigger/запуск, runs-log/очередь (per-source city-sweep runs + cancel), cron-расписание, and a per-source request delay — каждый скраппер своя задержка. Add /scrapers hub with global delay (lower-bound for all) + schedules summary table (enable/disable + next_run across all sources) + provider cards. Move cian-cookies (DMIR_AUTH upload + test-auth) into a section on the cian page; old /scrapers/cian-cookies route now client-redirects. Topbar gains a Скрапперы hub link. Backend cian/yandex city-sweep endpoints already exist on main (#860/#867). No backend changes in this PR — frontend-only unification. --- .../app/scrapers/_components/ScraperPage.tsx | 967 ++++++++++++++ .../frontend/src/app/scrapers/avito/page.tsx | 1146 ++++------------- .../src/app/scrapers/cian-cookies/page.tsx | 358 +---- .../frontend/src/app/scrapers/cian/page.tsx | 1053 ++++++++------- .../frontend/src/app/scrapers/page.tsx | 423 ++++++ .../frontend/src/app/scrapers/yandex/page.tsx | 709 ++++------ .../src/components/trade-in/Topbar.tsx | 9 +- 7 files changed, 2586 insertions(+), 2079 deletions(-) create mode 100644 tradein-mvp/frontend/src/app/scrapers/_components/ScraperPage.tsx create mode 100644 tradein-mvp/frontend/src/app/scrapers/page.tsx diff --git a/tradein-mvp/frontend/src/app/scrapers/_components/ScraperPage.tsx b/tradein-mvp/frontend/src/app/scrapers/_components/ScraperPage.tsx new file mode 100644 index 00000000..544087bb --- /dev/null +++ b/tradein-mvp/frontend/src/app/scrapers/_components/ScraperPage.tsx @@ -0,0 +1,967 @@ +"use client"; + +import { useEffect, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import "@/components/trade-in/trade-in.css"; +import { Topbar, type ActiveTab } from "@/components/trade-in/Topbar"; +import { apiFetch } from "@/lib/api"; + +// ── Shared Types ─────────────────────────────────────────────────────────── + +export interface ScrapeRunRow { + run_id: number; + source: string; + status: string; + params: Record | null; + counters: { + anchors_total?: number; + anchors_done?: number; + lots_fetched?: number; + lots_inserted?: number; + lots_updated?: number; + unique_houses?: number; + houses_enriched?: number; + houses_failed?: number; + detail_attempted?: number; + detail_enriched?: number; + detail_failed?: number; + errors_count?: number; + } | null; + error: string | null; + started_at: string | null; + finished_at: string | null; + heartbeat_at: string | null; +} + +export interface ScheduleConfig { + id: number; + source: string; + enabled: boolean; + window_start_hour: number; + window_end_hour: number; + default_params: { + pages_per_anchor?: number; + detail_top_n?: number; + request_delay_sec?: number; + enrich_houses?: boolean; + enrich_address?: boolean; + radius_m?: number; + }; + last_run_id: number | null; + last_run_at: string | null; + next_run_at: string | null; + updated_at: string | null; +} + +export interface ScraperSetting { + source: string; + request_delay_sec: number; + description: string | null; + updated_at: string | null; +} + +export interface CitySweepStartResp { + run_id: number; + status: string; + pages_per_anchor: number; + detail_top_n: number; +} + +// ── Shared sweep param types ─────────────────────────────────────────────── + +export interface SweepParamConfig { + /** has detail_top_n + enrich_houses params */ + hasDetailParams: boolean; + /** has enrich_address param (yandex) */ + hasEnrichAddress: boolean; + defaults: { + pages_per_anchor: number; + radius_m: number; + request_delay_sec: number; + detail_top_n?: number; + enrich_houses?: boolean; + enrich_address?: boolean; + }; +} + +// ── Shared API source name type ──────────────────────────────────────────── + +export type ScraperSource = "avito" | "cian" | "yandex"; +export type { ActiveTab }; +export type ScheduleSourceKey = + | "avito_city_sweep" + | "cian_city_sweep" + | "yandex_city_sweep"; + +// ── Shared Hooks ─────────────────────────────────────────────────────────── + +export function useStartCitySweep(source: ScraperSource) { + const qc = useQueryClient(); + return useMutation>({ + mutationFn: (input) => + apiFetch( + `/api/v1/admin/scrape/${source}-city-sweep`, + { + method: "POST", + body: JSON.stringify(input), + }, + ), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["city-sweep-runs", source] }); + }, + }); +} + +export function useCitySweepRuns(source: ScraperSource) { + return useQuery({ + queryKey: ["city-sweep-runs", source], + queryFn: () => + apiFetch( + `/api/v1/admin/scrape/${source}-city-sweep/runs?limit=10`, + ), + refetchInterval: 5_000, + staleTime: 0, + }); +} + +export function useCancelCitySweep(source: ScraperSource) { + const qc = useQueryClient(); + return useMutation< + { ok: boolean; run_id: number; cancelled: boolean }, + Error, + number + >({ + mutationFn: (run_id) => + apiFetch( + `/api/v1/admin/scrape/${source}-city-sweep/${run_id}/cancel`, + { method: "POST" }, + ), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["city-sweep-runs", source] }); + }, + }); +} + +export function useSchedules() { + return useQuery({ + queryKey: ["scrape-schedules"], + queryFn: () => + apiFetch("/api/v1/admin/scrape/schedules"), + refetchInterval: 30_000, + }); +} + +export function useUpdateSchedule() { + const qc = useQueryClient(); + return useMutation< + ScheduleConfig, + Error, + { + source: ScheduleSourceKey; + payload: { + enabled: boolean; + window_start_hour: number; + window_end_hour: number; + default_params: Record; + }; + } + >({ + mutationFn: ({ source, payload }) => + apiFetch(`/api/v1/admin/scrape/schedules/${source}`, { + method: "PUT", + body: JSON.stringify(payload), + }), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["scrape-schedules"] }); + }, + }); +} + +export function useScraperSettings() { + return useQuery({ + queryKey: ["scraper-settings"], + queryFn: () => + apiFetch<{ settings: ScraperSetting[] }>( + "/api/v1/admin/scraper-settings", + ).then((r) => + (r.settings ?? []).map((s) => ({ + ...s, + request_delay_sec: Number(s.request_delay_sec), + })), + ), + staleTime: 30_000, + refetchInterval: 60_000, + }); +} + +export function useUpdateScraperSetting() { + const qc = useQueryClient(); + return useMutation< + ScraperSetting, + Error, + { source: string; request_delay_sec: number } + >({ + mutationFn: ({ source, request_delay_sec }) => + apiFetch(`/api/v1/admin/scraper-settings/${source}`, { + method: "PUT", + body: JSON.stringify({ source, request_delay_sec }), + }), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["scraper-settings"] }); + }, + }); +} + +// ── Shared Helpers ───────────────────────────────────────────────────────── + +export function translateStatus(s: string): string { + switch (s) { + case "running": + return "выполняется"; + case "done": + return "готово"; + case "failed": + return "ошибка"; + case "cancelled": + return "отменено"; + case "zombie": + return "зомби"; + case "skipped": + return "пропущено"; + case "banned": + return "блок"; + default: + return s; + } +} + +export function formatTime(iso: string | null): string { + if (!iso) return "—"; + try { + return new Date(iso).toLocaleString("ru-RU", { + day: "2-digit", + month: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); + } catch { + return iso; + } +} + +export function utcToMsk(hour: number): number { + return (hour + 3) % 24; +} + +export function mskToUtc(hour: number): number { + return (hour - 3 + 24) % 24; +} + +function formatMskRange(utcStart: number, utcEnd: number): string { + const ms = utcToMsk(utcStart); + const me = utcToMsk(utcEnd); + return `${String(ms).padStart(2, "0")}:00–${String(me).padStart(2, "0")}:00 МСК`; +} + +// ── ResultPanel (reusable) ───────────────────────────────────────────────── + +export interface ResultPanelProps { + mut: { + data?: TData; + error: Error | null; + isPending: boolean; + isSuccess: boolean; + }; +} + +export function ResultPanel({ mut }: ResultPanelProps) { + if (mut.error) { + return ( +
+ Ошибка: {mut.error.message} +
+ ); + } + if (mut.isSuccess && mut.data) { + return ( +
+ 
{JSON.stringify(mut.data, null, 2)}
+
+ ); + } + return null; +} + +// ── Runs Log Section ─────────────────────────────────────────────────────── + +interface RunsLogSectionProps { + source: ScraperSource; +} + +function RunsLogSection({ source }: RunsLogSectionProps) { + const runsQ = useCitySweepRuns(source); + const cancelMut = useCancelCitySweep(source); + + return ( +
+

Очередь / Лог прогонов

+

+ Автообновление каждые 5 секунд. Показаны последние 10 прогонов. +

+
+

Recent runs (auto-refresh 5s)

+ {runsQ.isPending &&

Загрузка…

} + {runsQ.error && ( +

+ Ошибка загрузки runs: {runsQ.error.message} +

+ )} + {runsQ.data && runsQ.data.length === 0 && ( +

Нет запусков пока.

+ )} + {runsQ.data && runsQ.data.length > 0 && ( + + + + + + + + + + + + + + + + + {runsQ.data.map((r) => ( + + + + + + + + + + + + + ))} + +
#СтатусСтартHeartbeatAnchorsLotsHousesDetailErrorsДействие
{r.run_id} + + {translateStatus(r.status)} + + {formatTime(r.started_at)}{formatTime(r.heartbeat_at)} + {r.counters?.anchors_done ?? 0}/ + {r.counters?.anchors_total ?? 0} + + {r.counters?.lots_fetched ?? 0}{" "} + + (+{r.counters?.lots_inserted ?? 0}) + + + {r.counters?.houses_enriched ?? 0}/ + {r.counters?.unique_houses ?? 0} + + {r.counters?.detail_enriched ?? 0}/ + {r.counters?.detail_attempted ?? 0} + {r.counters?.errors_count ?? 0} + {r.status === "running" && ( + + )} +
+ )} +
+
+ ); +} + +// ── Cron Schedule Section ────────────────────────────────────────────────── + +interface CronScheduleSectionProps { + scheduleSource: ScheduleSourceKey; + paramConfig: SweepParamConfig; +} + +function CronScheduleSection({ + scheduleSource, + paramConfig, +}: CronScheduleSectionProps) { + const schedulesQ = useSchedules(); + const updateScheduleMut = useUpdateSchedule(); + + const [schEnabled, setSchEnabled] = useState(true); + const [schMskStart, setSchMskStart] = useState(5); + const [schMskEnd, setSchMskEnd] = useState(8); + const [schPages, setSchPages] = useState( + paramConfig.defaults.pages_per_anchor, + ); + const [schTopN, setSchTopN] = useState(paramConfig.defaults.detail_top_n ?? 0); + const [schDelay, setSchDelay] = useState( + paramConfig.defaults.request_delay_sec, + ); + const [schRadius, setSchRadius] = useState(paramConfig.defaults.radius_m); + const [schEnrichHouses, setSchEnrichHouses] = useState( + paramConfig.defaults.enrich_houses ?? false, + ); + const [schEnrichAddress, setSchEnrichAddress] = useState( + paramConfig.defaults.enrich_address ?? false, + ); + + useEffect(() => { + const sweep = schedulesQ.data?.find((s) => s.source === scheduleSource); + if (!sweep) return; + setSchEnabled(sweep.enabled); + setSchMskStart(utcToMsk(sweep.window_start_hour)); + setSchMskEnd(utcToMsk(sweep.window_end_hour)); + const p = sweep.default_params ?? {}; + if (typeof p.pages_per_anchor === "number") + setSchPages(p.pages_per_anchor); + if (typeof p.detail_top_n === "number") setSchTopN(p.detail_top_n); + if (typeof p.request_delay_sec === "number") + setSchDelay(p.request_delay_sec); + if (typeof p.radius_m === "number") setSchRadius(p.radius_m); + if (typeof p.enrich_houses === "boolean") setSchEnrichHouses(p.enrich_houses); + if (typeof p.enrich_address === "boolean") + setSchEnrichAddress(p.enrich_address); + }, [schedulesQ.data, scheduleSource]); + + const currentSweep = schedulesQ.data?.find( + (s) => s.source === scheduleSource, + ); + + return ( +
+

Cron-расписание автозапуска (in-app scheduler)

+

+ Backend периодически (каждую минуту) проверяет это расписание и + запускает city sweep автоматически в случайное время в указанном окне. + Без SSH, без crontab — всё в БД и UI. +

+ + {schedulesQ.isPending &&

Загрузка…

} + {schedulesQ.error && ( +

+ Ошибка загрузки: {schedulesQ.error.message} +

+ )} + + {currentSweep && ( +
+
+ Статус: + + {currentSweep.enabled ? "включено" : "отключено"} + +
+
+ Окно (МСК): + + {formatMskRange( + currentSweep.window_start_hour, + currentSweep.window_end_hour, + )} + + + ({String(currentSweep.window_start_hour).padStart(2, "0")}:00– + {String(currentSweep.window_end_hour).padStart(2, "0")}:00 UTC) + +
+
+ Следующий запуск: + {formatTime(currentSweep.next_run_at)} +
+
+ Последний запуск: + + {currentSweep.last_run_at + ? formatTime(currentSweep.last_run_at) + : "—"} + {currentSweep.last_run_id && + ` (run #${currentSweep.last_run_id})`} + +
+
+ )} + +
{ + e.preventDefault(); + const defaultParams: Record = { + pages_per_anchor: schPages, + request_delay_sec: schDelay, + radius_m: schRadius, + }; + if (paramConfig.hasDetailParams) { + defaultParams.detail_top_n = schTopN; + defaultParams.enrich_houses = schEnrichHouses; + } + if (paramConfig.hasEnrichAddress) { + defaultParams.enrich_address = schEnrichAddress; + } + updateScheduleMut.mutate({ + source: scheduleSource, + payload: { + enabled: schEnabled, + window_start_hour: mskToUtc(schMskStart), + window_end_hour: mskToUtc(schMskEnd), + default_params: defaultParams, + }, + }); + }} + > + + + + + {paramConfig.hasDetailParams && ( + <> + + + )} + + + {paramConfig.hasDetailParams && ( + + )} + {paramConfig.hasEnrichAddress && ( + + )} + + + + {updateScheduleMut.error && ( +
+ Ошибка: {updateScheduleMut.error.message} +
+ )} + {updateScheduleMut.isSuccess && updateScheduleMut.data && ( +
+ Расписание обновлено. Next run:{" "} + {formatTime(updateScheduleMut.data.next_run_at)} +
+ )} +
+ ); +} + +// ── Per-source Delay Section ─────────────────────────────────────────────── + +interface PerSourceDelaySectionProps { + source: ScraperSource; + delayMin?: number; + delayMax?: number; +} + +function PerSourceDelaySection({ + source, + delayMin = 1, + delayMax = 60, +}: PerSourceDelaySectionProps) { + const settingsQ = useScraperSettings(); + const updateMut = useUpdateScraperSetting(); + + const currentSetting = settingsQ.data?.find((s) => s.source === source); + const [delayInput, setDelayInput] = useState("5.0"); + const [initialized, setInitialized] = useState(false); + + useEffect(() => { + if (currentSetting && !initialized) { + setDelayInput(String(currentSetting.request_delay_sec)); + setInitialized(true); + } + }, [currentSetting, initialized]); + + return ( +
+

Задержка запросов (этот скраппер)

+

+ Задержка применяется только к скрапперу {source}. + Изменение вступает в силу немедленно без рестарта backend. Глобальный + нижний порог настраивается на{" "} + странице хаба скрапперов. +

+ + {settingsQ.isPending && ( +

Загрузка настроек…

+ )} + {settingsQ.error && ( +

+ Ошибка загрузки: {settingsQ.error.message} +

+ )} + + {currentSetting && ( +
+
+ Текущая задержка: + + {currentSetting.request_delay_sec} сек + +
+ {currentSetting.updated_at && ( +
+ Обновлено: + {formatTime(currentSetting.updated_at)} +
+ )} + {currentSetting.description && ( +
+ Описание: + {currentSetting.description} +
+ )} +
+ )} + +
{ + e.preventDefault(); + updateMut.mutate({ + source, + request_delay_sec: parseFloat(delayInput), + }); + }} + > + + +
+ + {updateMut.error && ( +
+ Ошибка: {updateMut.error.message} +
+ )} + {updateMut.isSuccess && updateMut.data && ( +
+ Сохранено. Задержка:{" "} + + {Number(updateMut.data.request_delay_sec).toFixed(1)} сек + +
+ )} +
+ ); +} + +// ── City Sweep Section (shared trigger) ─────────────────────────────────── + +interface CitySweepSectionProps { + source: ScraperSource; + paramConfig: SweepParamConfig; +} + +function CitySweepSection({ source, paramConfig }: CitySweepSectionProps) { + const sweepMut = useStartCitySweep(source); + + const [sweepPages, setSweepPages] = useState( + String(paramConfig.defaults.pages_per_anchor), + ); + const [sweepTopN, setSweepTopN] = useState( + String(paramConfig.defaults.detail_top_n ?? 0), + ); + const [sweepDelay, setSweepDelay] = useState( + String(paramConfig.defaults.request_delay_sec), + ); + const [sweepRadius, setSweepRadius] = useState( + String(paramConfig.defaults.radius_m), + ); + const [sweepEnrichHouses, setSweepEnrichHouses] = useState( + paramConfig.defaults.enrich_houses ?? false, + ); + const [sweepEnrichAddress, setSweepEnrichAddress] = useState( + paramConfig.defaults.enrich_address ?? false, + ); + + return ( +
+

Полный обход ЕКБ (city sweep)

+

+ Iterate anchors × N pages → save listings → enrich. Запускается в + background. Можно отменить в логе прогонов ниже. +

+
{ + e.preventDefault(); + const body: Record = { + pages_per_anchor: parseInt(sweepPages, 10), + request_delay_sec: parseFloat(sweepDelay), + radius_m: parseInt(sweepRadius, 10), + }; + if (paramConfig.hasDetailParams) { + body.detail_top_n = parseInt(sweepTopN, 10); + body.enrich_houses = sweepEnrichHouses; + } + if (paramConfig.hasEnrichAddress) { + body.enrich_address = sweepEnrichAddress; + } + sweepMut.mutate(body); + }} + > + + {paramConfig.hasDetailParams && ( + + )} + + + {paramConfig.hasDetailParams && ( + + )} + {paramConfig.hasEnrichAddress && ( + + )} + + + + {sweepMut.error && ( +
+ Ошибка: {sweepMut.error.message} +
+ )} + {sweepMut.isSuccess && sweepMut.data && ( +
+ 
{JSON.stringify(sweepMut.data, null, 2)}
+
+ )} +
+ ); +} + +// ── ScraperPage config ───────────────────────────────────────────────────── + +export interface ScraperPageConfig { + source: ScraperSource; + activeTab: ActiveTab; + displayName: string; + subtitle: string; + scheduleSource: ScheduleSourceKey; + paramConfig: SweepParamConfig; + delayMin?: number; + delayMax?: number; + /** Extra trigger forms rendered inside the "Запуск" section */ + extraTriggers?: React.ReactNode; + /** Extra tabs/sections rendered after the standard 4 sections */ + extraSections?: React.ReactNode; +} + +// ── ScraperPage (main export) ────────────────────────────────────────────── + +export function ScraperPage({ config }: { config: ScraperPageConfig }) { + const { + source, + activeTab, + displayName, + subtitle, + scheduleSource, + paramConfig, + delayMin, + delayMax, + extraTriggers, + extraSections, + } = config; + + return ( + <> + +
+

{displayName} — admin триггер

+

{subtitle}

+ + {/* Section 1: Запуск — city sweep + extra provider triggers */} +
+

Запуск

+ {extraTriggers} + +
+ + {/* Section 2: Runs log */} + + + {/* Section 3: Cron schedule */} + + + {/* Section 4: Per-source delay */} + + + {/* Extra sections (e.g. cookies tab for cian) */} + {extraSections} +
+ + ); +} diff --git a/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx b/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx index 0b961ab5..0f42b935 100644 --- a/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx +++ b/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx @@ -1,19 +1,16 @@ "use client"; -import { useEffect, useState } from "react"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useState } from "react"; +import { useMutation } from "@tanstack/react-query"; -import "@/components/trade-in/trade-in.css"; -import { Topbar } from "@/components/trade-in/Topbar"; import { apiFetch } from "@/lib/api"; +import { + ResultPanel, + ScraperPage, + type ScraperPageConfig, +} from "@/app/scrapers/_components/ScraperPage"; -// ── Types ────────────────────────────────────────────────────────────────── - -interface ScrapeAroundInput { - lat: number; - lon: number; - radius_m: number; -} +// ── Avito-specific trigger types + hooks ─────────────────────────────────── interface ScrapeAroundResp { total_fetched: number; @@ -39,18 +36,6 @@ interface DetailTriggerResp { updated: boolean; } -interface ImvInput { - address: string; - rooms: number; - area_m2: number; - floor: number; - floor_at_home: number; - house_type: string; - renovation_type: string; - has_balcony: boolean; - has_loggia: boolean; -} - interface ImvResp { ok: boolean; cache_key: string; @@ -61,10 +46,12 @@ interface ImvResp { history_saved: number; } -// ── Mutations ────────────────────────────────────────────────────────────── - function useScrapeAround() { - return useMutation({ + return useMutation< + ScrapeAroundResp, + Error, + { lat: number; lon: number; radius_m: number } + >({ mutationFn: (input) => apiFetch("/api/v1/admin/scrape", { method: "POST", @@ -94,7 +81,21 @@ function useScrapeDetail() { } function useScrapeImv() { - return useMutation({ + return useMutation< + ImvResp, + Error, + { + address: string; + rooms: number; + area_m2: number; + floor: number; + floor_at_home: number; + house_type: string; + renovation_type: string; + has_balcony: boolean; + has_loggia: boolean; + } + >({ mutationFn: (input) => { const qs = new URLSearchParams({ address: input.address, @@ -115,181 +116,9 @@ function useScrapeImv() { }); } -// ── City Sweep types + hooks ─────────────────────────────────────────────── +// ── Extra triggers ───────────────────────────────────────────────────────── -interface CitySweepInput { - pages_per_anchor: number; - detail_top_n: number; - request_delay_sec: number; - enrich_houses: boolean; - radius_m: number; -} - -interface CitySweepStartResp { - run_id: number; - status: string; - pages_per_anchor: number; - detail_top_n: number; -} - -interface CitySweepCounters { - anchors_total?: number; - anchors_done?: number; - lots_fetched?: number; - lots_inserted?: number; - lots_updated?: number; - unique_houses?: number; - houses_enriched?: number; - houses_failed?: number; - detail_attempted?: number; - detail_enriched?: number; - detail_failed?: number; - errors_count?: number; -} - -interface ScrapeRunRow { - run_id: number; - source: string; - status: string; - params: Record | null; - counters: CitySweepCounters | null; - error: string | null; - started_at: string | null; - finished_at: string | null; - heartbeat_at: string | null; -} - -function useStartCitySweep() { - const qc = useQueryClient(); - return useMutation({ - mutationFn: (input) => - apiFetch("/api/v1/admin/scrape/avito-city-sweep", { - method: "POST", - body: JSON.stringify(input), - }), - onSuccess: () => { - qc.invalidateQueries({ queryKey: ["city-sweep-runs"] }); - }, - }); -} - -function useCitySweepRuns() { - return useQuery({ - queryKey: ["city-sweep-runs"], - queryFn: () => - apiFetch( - "/api/v1/admin/scrape/avito-city-sweep/runs?limit=10", - ), - refetchInterval: 5_000, - staleTime: 0, - }); -} - -function useCancelCitySweep() { - const qc = useQueryClient(); - return useMutation< - { ok: boolean; run_id: number; cancelled: boolean }, - Error, - number - >({ - mutationFn: (run_id) => - apiFetch( - `/api/v1/admin/scrape/avito-city-sweep/${run_id}/cancel`, - { method: "POST" }, - ), - onSuccess: () => { - qc.invalidateQueries({ queryKey: ["city-sweep-runs"] }); - }, - }); -} - -// ── Schedule types + hooks ───────────────────────────────────────────────── - -interface ScheduleConfig { - id: number; - source: string; - enabled: boolean; - window_start_hour: number; - window_end_hour: number; - default_params: { - pages_per_anchor?: number; - detail_top_n?: number; - request_delay_sec?: number; - enrich_houses?: boolean; - radius_m?: number; - }; - last_run_id: number | null; - last_run_at: string | null; - next_run_at: string | null; - updated_at: string | null; -} - -interface ScheduleConfigUpdate { - enabled: boolean; - window_start_hour: number; - window_end_hour: number; - default_params: Record; -} - -function useSchedules() { - return useQuery({ - queryKey: ["scrape-schedules"], - queryFn: () => - apiFetch("/api/v1/admin/scrape/schedules"), - refetchInterval: 30_000, - }); -} - -function useUpdateSchedule() { - const qc = useQueryClient(); - return useMutation< - ScheduleConfig, - Error, - { source: string; payload: ScheduleConfigUpdate } - >({ - mutationFn: ({ source, payload }) => - apiFetch(`/api/v1/admin/scrape/schedules/${source}`, { - method: "PUT", - body: JSON.stringify(payload), - }), - onSuccess: () => { - qc.invalidateQueries({ queryKey: ["scrape-schedules"] }); - }, - }); -} - -// ── Result panel ─────────────────────────────────────────────────────────── - -interface ResultPanelProps { - mut: { - data?: TData; - error: Error | null; - isPending: boolean; - isSuccess: boolean; - }; -} - -function ResultPanel({ mut }: ResultPanelProps) { - if (mut.error) { - return ( -
- Ошибка: {mut.error.message} -
- ); - } - if (mut.isSuccess && mut.data) { - return ( -
- 
{JSON.stringify(mut.data, null, 2)}
-
- ); - } - return null; -} - -// ── Page ─────────────────────────────────────────────────────────────────── - -export default function AvitoScraperPage() { +function AvitoExtraTriggers() { // 1. Around-point const [lat, setLat] = useState("56.8400"); const [lon, setLon] = useState("60.6050"); @@ -322,682 +151,265 @@ export default function AvitoScraperPage() { const [imvLoggia, setImvLoggia] = useState(false); const imvMut = useScrapeImv(); - // City sweep state - const [sweepPages, setSweepPages] = useState("3"); - const [sweepTopN, setSweepTopN] = useState("20"); - const [sweepDelay, setSweepDelay] = useState("7"); - const [sweepRadius, setSweepRadius] = useState("1500"); - const [sweepEnrichHouses, setSweepEnrichHouses] = useState(true); - const sweepMut = useStartCitySweep(); - const sweepRuns = useCitySweepRuns(); - const cancelMut = useCancelCitySweep(); - - // Scheduler state - const schedulesQ = useSchedules(); - const updateScheduleMut = useUpdateSchedule(); - - const [schEnabled, setSchEnabled] = useState(true); - const [schMskStart, setSchMskStart] = useState(5); - const [schMskEnd, setSchMskEnd] = useState(8); - const [schPages, setSchPages] = useState(3); - const [schTopN, setSchTopN] = useState(20); - const [schDelay, setSchDelay] = useState(7); - const [schRadius, setSchRadius] = useState(1500); - const [schEnrich, setSchEnrich] = useState(true); - - useEffect(() => { - const sweep = schedulesQ.data?.find((s) => s.source === "avito_city_sweep"); - if (!sweep) return; - setSchEnabled(sweep.enabled); - setSchMskStart(utcToMsk(sweep.window_start_hour)); - setSchMskEnd(utcToMsk(sweep.window_end_hour)); - const p = sweep.default_params || {}; - if (typeof p.pages_per_anchor === "number") setSchPages(p.pages_per_anchor); - if (typeof p.detail_top_n === "number") setSchTopN(p.detail_top_n); - if (typeof p.request_delay_sec === "number") setSchDelay(p.request_delay_sec); - if (typeof p.radius_m === "number") setSchRadius(p.radius_m); - if (typeof p.enrich_houses === "boolean") setSchEnrich(p.enrich_houses); - }, [schedulesQ.data]); - return ( <> - -
-

Avito скрапер — admin триггер

-

- Ручной запуск sub-pipeline'ов AvitoScraper v2. Для бесплатной - разработки и теста после изменений. + {/* 1. Search around point */} +

+

1. Search around (общий /admin/scrape)

+

+ Запускает классический cron-pipeline для одного якоря: search → + save_listings. Использовать как baseline.

+
{ + e.preventDefault(); + aroundMut.mutate({ + lat: parseFloat(lat), + lon: parseFloat(lon), + radius_m: parseInt(radius, 10), + }); + }} + > + + + + +
+ +
- {/* 1. Search around point */} -
-

1. Search around (общий /admin/scrape)

-

- Запускает классический cron-pipeline для одного якоря: search → - save_listings. Использовать как baseline. -

-
{ - e.preventDefault(); - aroundMut.mutate({ - lat: parseFloat(lat), - lon: parseFloat(lon), - radius_m: parseInt(radius, 10), - }); - }} - > - - - - -
- -
+ {/* 2. Houses Catalog */} +
+

2. Houses Catalog enrichment

+

+ Парсит /catalog/houses/<slug>/<id> — характеристики + дома, отзывы, history. +

+
{ + e.preventDefault(); + houseMut.mutate({ house_url: houseUrl }); + }} + > + + +
+ +
- {/* 2. Houses Catalog */} -
-

2. Houses Catalog enrichment

-

- Парсит /catalog/houses/<slug>/<id> — характеристики - дома, отзывы, history. -

-
{ - e.preventDefault(); - houseMut.mutate({ house_url: houseUrl }); - }} - > - - -
- -
+ {/* 3. Detail */} +
+

3. Detail page enrichment

+

+ Парсит /ekaterinburg/kvartiry/<...> — 30+ полей (kitchen_area, + owners_count, etc). UPDATE listings WHERE source_id (listing должен + уже существовать). +

+
{ + e.preventDefault(); + detailMut.mutate({ item_url: itemUrl }); + }} + > + + +
+ +
- {/* 3. Detail */} -
-

3. Detail page enrichment

-

- Парсит /ekaterinburg/kvartiry/<...> — 30+ полей - (kitchen_area, owners_count, etc). UPDATE listings WHERE source_id - (listing должен уже существовать). -

-
{ - e.preventDefault(); - detailMut.mutate({ item_url: itemUrl }); - }} - > - - -
- -
- - {/* 4. IMV evaluation */} -
-

4. IMV evaluation (debug, без cache)

-

- Прямой вызов Avito IMV API (2 HTTP requests). Production вызов из - estimator on-demand с 24h cache; здесь — для debug. -

-
{ - e.preventDefault(); - imvMut.mutate({ - address: imvAddress, - rooms: parseInt(imvRooms, 10), - area_m2: parseFloat(imvArea), - floor: parseInt(imvFloor, 10), - floor_at_home: parseInt(imvTotalFloors, 10), - house_type: imvHouseType, - renovation_type: imvRenovation, - has_balcony: imvBalcony, - has_loggia: imvLoggia, - }); - }} - > - - - - - - - - - - -
- -
- - {/* 5. City sweep ЕКБ */} -
-

5. Полный обход ЕКБ (auto sweep)

-

- Iterate 5 anchors × N pages → save listings → enrich houses + detail top-N. - Запускается в background (~15-30 мин для defaults). Можно отменить через - «Отменить» в списке ниже. Cron автоматически запускает каждый день в случайное - время 02:00-05:00 UTC. -

-
{ - e.preventDefault(); - sweepMut.mutate({ - pages_per_anchor: parseInt(sweepPages, 10), - detail_top_n: parseInt(sweepTopN, 10), - request_delay_sec: parseFloat(sweepDelay), - enrich_houses: sweepEnrichHouses, - radius_m: parseInt(sweepRadius, 10), - }); - }} - > - - - - - - -
- - {sweepMut.error && ( -
- Ошибка: {sweepMut.error.message} -
- )} - {sweepMut.isSuccess && sweepMut.data && ( -
- 
{JSON.stringify(sweepMut.data, null, 2)}
-
- )} - - {/* Recent runs list */} -
-

Recent runs (auto-refresh 5s)

- {sweepRuns.isPending &&

Загрузка…

} - {sweepRuns.error && ( -

- Ошибка загрузки runs: {sweepRuns.error.message} -

- )} - {sweepRuns.data && sweepRuns.data.length === 0 && ( -

Нет запусков пока.

- )} - {sweepRuns.data && sweepRuns.data.length > 0 && ( - - - - - - - - - - - - - - - - - {sweepRuns.data.map((r) => ( - - - - - - - - - - - - - ))} - -
#СтатусСтартHeartbeatAnchorsLotsHousesDetailErrorsДействие
{r.run_id} - - {translateStatus(r.status)} - - {formatTime(r.started_at)}{formatTime(r.heartbeat_at)} - {r.counters?.anchors_done ?? 0}/{r.counters?.anchors_total ?? 0} - - {r.counters?.lots_fetched ?? 0}{" "} - - (+{r.counters?.lots_inserted ?? 0}) - - - {r.counters?.houses_enriched ?? 0}/{r.counters?.unique_houses ?? 0} - - {r.counters?.detail_enriched ?? 0}/{r.counters?.detail_attempted ?? 0} - {r.counters?.errors_count ?? 0} - {r.status === "running" && ( - - )} -
- )} -
-
- - {/* 6. Расписание автозапуска */} -
-

6. Расписание автозапуска (in-app scheduler)

-

- Backend периодически (каждую минуту) проверяет это расписание и запускает - city sweep автоматически в случайное время в указанном окне. Без SSH, без - crontab — всё в БД и UI. -

- - {schedulesQ.isPending &&

Загрузка…

} - {schedulesQ.error && ( -

- Ошибка загрузки: {schedulesQ.error.message} -

- )} - - {schedulesQ.data && ( - <> - {/* Status preview */} - {(() => { - const sweep = schedulesQ.data.find( - (s) => s.source === "avito_city_sweep", - ); - if (!sweep) return null; - return ( -
-
- Status: - - {sweep.enabled ? "включено" : "отключено"} - -
-
- Окно (МСК): - - {formatMskRange( - sweep.window_start_hour, - sweep.window_end_hour, - )} - - - ({String(sweep.window_start_hour).padStart(2, "0")}:00– - {String(sweep.window_end_hour).padStart(2, "0")}:00 UTC) - -
-
- - Следующий запуск: - - {formatTime(sweep.next_run_at)} -
-
- - Последний запуск: - - - {sweep.last_run_at ? formatTime(sweep.last_run_at) : "—"} - {sweep.last_run_id && ` (run #${sweep.last_run_id})`} - -
-
- ); - })()} - - {/* Edit form */} -
{ - e.preventDefault(); - updateScheduleMut.mutate({ - source: "avito_city_sweep", - payload: { - enabled: schEnabled, - window_start_hour: mskToUtc(schMskStart), - window_end_hour: mskToUtc(schMskEnd), - default_params: { - pages_per_anchor: schPages, - detail_top_n: schTopN, - request_delay_sec: schDelay, - enrich_houses: schEnrich, - radius_m: schRadius, - }, - }, - }); - }} - > - - - - - - - - - -
- - {updateScheduleMut.error && ( -
- Ошибка: {updateScheduleMut.error.message} -
- )} - {updateScheduleMut.isSuccess && updateScheduleMut.data && ( -
- ✓ Расписание обновлено. Next run:{" "} - {formatTime(updateScheduleMut.data.next_run_at)} -
- )} - - )} -
-
+ {/* 4. IMV evaluation */} +
+

4. IMV evaluation (debug, без cache)

+

+ Прямой вызов Avito IMV API (2 HTTP requests). Production вызов из + estimator on-demand с 24h cache; здесь — для debug. +

+
{ + e.preventDefault(); + imvMut.mutate({ + address: imvAddress, + rooms: parseInt(imvRooms, 10), + area_m2: parseFloat(imvArea), + floor: parseInt(imvFloor, 10), + floor_at_home: parseInt(imvTotalFloors, 10), + house_type: imvHouseType, + renovation_type: imvRenovation, + has_balcony: imvBalcony, + has_loggia: imvLoggia, + }); + }} + > + + + + + + + + + + +
+ +
); } -// ── Helpers ──────────────────────────────────────────────────────────────── +// ── Page config ──────────────────────────────────────────────────────────── -function translateStatus(s: string): string { - switch (s) { - case "running": - return "выполняется"; - case "done": - return "готово"; - case "failed": - return "ошибка"; - case "cancelled": - return "отменено"; - case "zombie": - return "зомби"; - case "skipped": - return "пропущено"; - case "banned": - return "блок"; - default: - return s; - } -} +const AVITO_CONFIG: ScraperPageConfig = { + source: "avito", + activeTab: "avito", + displayName: "Avito скрапер", + subtitle: + "Ручной запуск sub-pipeline'ов AvitoScraper v2. Для бесплатной разработки и теста после изменений.", + scheduleSource: "avito_city_sweep", + paramConfig: { + hasDetailParams: true, + hasEnrichAddress: false, + defaults: { + pages_per_anchor: 3, + detail_top_n: 20, + request_delay_sec: 7, + radius_m: 1500, + enrich_houses: true, + }, + }, + delayMin: 3, + delayMax: 30, +}; -function formatTime(iso: string | null): string { - if (!iso) return "—"; - try { - return new Date(iso).toLocaleString("ru-RU", { - day: "2-digit", - month: "2-digit", - hour: "2-digit", - minute: "2-digit", - second: "2-digit", - }); - } catch { - return iso; - } -} - -function utcToMsk(hour: number): number { - return (hour + 3) % 24; -} - -function mskToUtc(hour: number): number { - return (hour - 3 + 24) % 24; -} - -function formatMskRange(utcStart: number, utcEnd: number): string { - const ms = utcToMsk(utcStart); - const me = utcToMsk(utcEnd); - return `${String(ms).padStart(2, "0")}:00–${String(me).padStart(2, "0")}:00 МСК`; +export default function AvitoScraperPage() { + return ( + , + }} + /> + ); } diff --git a/tradein-mvp/frontend/src/app/scrapers/cian-cookies/page.tsx b/tradein-mvp/frontend/src/app/scrapers/cian-cookies/page.tsx index d7042f21..6bd4c037 100644 --- a/tradein-mvp/frontend/src/app/scrapers/cian-cookies/page.tsx +++ b/tradein-mvp/frontend/src/app/scrapers/cian-cookies/page.tsx @@ -1,351 +1,21 @@ "use client"; -import { useState } from "react"; -import { useMutation, useQuery } from "@tanstack/react-query"; +import { useEffect } from "react"; +import { useRouter } from "next/navigation"; -import "@/components/trade-in/trade-in.css"; -import { Topbar } from "@/components/trade-in/Topbar"; -import { apiFetch } from "@/lib/api"; - -interface UploadResponse { - ok?: boolean; - userId?: number; - cookieCount?: number; - detail?: string; -} - -interface TestAuthResponse { - authenticated: boolean; - userId?: number; - reason?: string; -} - -interface CookieEditorEntry { - name: string; - value: string | number | boolean; - [key: string]: unknown; -} - -function convertCookiesFormat(input: string): Record { - const parsed: unknown = JSON.parse(input.trim()); - if (Array.isArray(parsed)) { - // Cookie-Editor array format: [{name, value, domain, ...}, ...] - return (parsed as CookieEditorEntry[]).reduce>( - (acc, c) => { - if ( - typeof c === "object" && - c !== null && - "name" in c && - "value" in c - ) { - acc[String(c.name)] = String(c.value); - } - return acc; - }, - {}, - ); - } - if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) { - // Already a dict: {"name1": "val1", "name2": "val2"} - return Object.fromEntries( - Object.entries(parsed as Record).map(([k, v]) => [ - k, - String(v), - ]), - ); - } - throw new Error("Expected JSON array or plain object"); -} - -function parsePreview(raw: string): { - cookies: Record | null; - error: string | null; -} { - if (!raw.trim()) return { cookies: null, error: null }; - try { - const cookies = convertCookiesFormat(raw); - return { cookies, error: null }; - } catch (e) { - return { cookies: null, error: e instanceof Error ? e.message : String(e) }; - } -} - -export default function CianCookiesPage() { - const [rawInput, setRawInput] = useState(""); - const [showInstructions, setShowInstructions] = useState(false); - - const preview = parsePreview(rawInput); - const previewKeys = preview.cookies ? Object.keys(preview.cookies) : []; - - const uploadMutation = useMutation({ - mutationFn: () => { - if (!preview.cookies) { - throw new Error("Invalid JSON — fix the input first"); - } - return apiFetch( - "/api/v1/admin/scrape/cian/upload-cookies", - { - method: "POST", - body: JSON.stringify(preview.cookies), - }, - ); - }, - onSuccess: () => setRawInput(""), - }); - - const testAuthQuery = useQuery({ - queryKey: ["cian-test-auth"], - queryFn: () => - apiFetch("/api/v1/admin/scrape/cian/test-auth"), - enabled: false, // only on explicit button click - retry: false, - }); - - const canUpload = - !!preview.cookies && previewKeys.length > 0 && !uploadMutation.isPending; +/** + * /scrapers/cian-cookies redirects to /scrapers/cian where cookie management + * is now integrated as a dedicated section at the bottom of the Cian page. + */ +export default function CianCookiesRedirect() { + const router = useRouter(); + useEffect(() => { + router.replace("/scrapers/cian"); + }, [router]); return ( - <> - -
-

Cian — обновление cookies

-

- Позволяет обновить сессию Cian для скрапера оценщика без перезапуска - backend. Endpoints:{" "} - POST /api/v1/admin/scrape/cian/upload-cookies ·{" "} - GET /api/v1/admin/scrape/cian/test-auth -

- - {/* Instructions collapsible */} -
- - {showInstructions && ( -
    -
  1. - Открой{" "} - - cian.ru - {" "} - и войди под нужным аккаунтом. -
    2. -
  2. - Установи расширение Cookie-Editor{" "} - (Chrome/Firefox) или открой DevTools → Application → - Cookies → www.cian.ru. -
    3. -
  3. - Cookie-Editor: нажми Export → Export as JSON → - скопируй массив{" "} - [{"{"}name, value, ...{"}"}]. -
    4. -
  4. - DevTools (ручной способ): собери объект{" "} - - {"{"} - {'"'}name1{'"'}: {'"'}val1{'"'}, ...{"}"} - - . -
    5. -
  5. - Вставь JSON в поле ниже — формат определится автоматически. -
    6. -
  6. - Убедись что preview показывает нужные ключи, затем нажми - Upload. -
    7. -
- )} -
- - {/* Cookie input */} -
-

Cookie JSON

-

- Вставьте Cookie-Editor array или plain dict. Формат определяется - автоматически. -

-
-