feat(tradein): unify scraper admin pages (avito/cian/yandex) + /scrapers hub (#878)
All checks were successful
Deploy Trade-In / changes (push) Successful in 5s
Deploy Trade-In / test (push) Has been skipped
Deploy Trade-In / build-backend (push) Has been skipped
Deploy Trade-In / build-frontend (push) Successful in 1m37s
Deploy Trade-In / deploy (push) Successful in 36s

Co-authored-by: bot-frontend <bot-frontend@gendsgn.local>
Co-committed-by: bot-frontend <bot-frontend@gendsgn.local>
This commit is contained in:
bot-frontend 2026-05-31 09:31:47 +00:00 committed by bot-reviewer
parent d9b8047a7d
commit ee8f72da39
7 changed files with 2586 additions and 2079 deletions

View file

@ -0,0 +1,967 @@
"use client";
import { useEffect, useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import "@/components/trade-in/trade-in.css";
import { Topbar, type ActiveTab } from "@/components/trade-in/Topbar";
import { apiFetch } from "@/lib/api";
// ── Shared Types ───────────────────────────────────────────────────────────
export interface ScrapeRunRow {
run_id: number;
source: string;
status: string;
params: Record<string, unknown> | null;
counters: {
anchors_total?: number;
anchors_done?: number;
lots_fetched?: number;
lots_inserted?: number;
lots_updated?: number;
unique_houses?: number;
houses_enriched?: number;
houses_failed?: number;
detail_attempted?: number;
detail_enriched?: number;
detail_failed?: number;
errors_count?: number;
} | null;
error: string | null;
started_at: string | null;
finished_at: string | null;
heartbeat_at: string | null;
}
export interface ScheduleConfig {
id: number;
source: string;
enabled: boolean;
window_start_hour: number;
window_end_hour: number;
default_params: {
pages_per_anchor?: number;
detail_top_n?: number;
request_delay_sec?: number;
enrich_houses?: boolean;
enrich_address?: boolean;
radius_m?: number;
};
last_run_id: number | null;
last_run_at: string | null;
next_run_at: string | null;
updated_at: string | null;
}
export interface ScraperSetting {
source: string;
request_delay_sec: number;
description: string | null;
updated_at: string | null;
}
export interface CitySweepStartResp {
run_id: number;
status: string;
pages_per_anchor: number;
detail_top_n: number;
}
// ── Shared sweep param types ───────────────────────────────────────────────
export interface SweepParamConfig {
/** has detail_top_n + enrich_houses params */
hasDetailParams: boolean;
/** has enrich_address param (yandex) */
hasEnrichAddress: boolean;
defaults: {
pages_per_anchor: number;
radius_m: number;
request_delay_sec: number;
detail_top_n?: number;
enrich_houses?: boolean;
enrich_address?: boolean;
};
}
// ── Shared API source name type ────────────────────────────────────────────
export type ScraperSource = "avito" | "cian" | "yandex";
export type { ActiveTab };
export type ScheduleSourceKey =
| "avito_city_sweep"
| "cian_city_sweep"
| "yandex_city_sweep";
// ── Shared Hooks ───────────────────────────────────────────────────────────
export function useStartCitySweep(source: ScraperSource) {
const qc = useQueryClient();
return useMutation<CitySweepStartResp, Error, Record<string, unknown>>({
mutationFn: (input) =>
apiFetch<CitySweepStartResp>(
`/api/v1/admin/scrape/${source}-city-sweep`,
{
method: "POST",
body: JSON.stringify(input),
},
),
onSuccess: () => {
qc.invalidateQueries({ queryKey: ["city-sweep-runs", source] });
},
});
}
export function useCitySweepRuns(source: ScraperSource) {
return useQuery<ScrapeRunRow[]>({
queryKey: ["city-sweep-runs", source],
queryFn: () =>
apiFetch<ScrapeRunRow[]>(
`/api/v1/admin/scrape/${source}-city-sweep/runs?limit=10`,
),
refetchInterval: 5_000,
staleTime: 0,
});
}
export function useCancelCitySweep(source: ScraperSource) {
const qc = useQueryClient();
return useMutation<
{ ok: boolean; run_id: number; cancelled: boolean },
Error,
number
>({
mutationFn: (run_id) =>
apiFetch(
`/api/v1/admin/scrape/${source}-city-sweep/${run_id}/cancel`,
{ method: "POST" },
),
onSuccess: () => {
qc.invalidateQueries({ queryKey: ["city-sweep-runs", source] });
},
});
}
export function useSchedules() {
return useQuery<ScheduleConfig[]>({
queryKey: ["scrape-schedules"],
queryFn: () =>
apiFetch<ScheduleConfig[]>("/api/v1/admin/scrape/schedules"),
refetchInterval: 30_000,
});
}
export function useUpdateSchedule() {
const qc = useQueryClient();
return useMutation<
ScheduleConfig,
Error,
{
source: ScheduleSourceKey;
payload: {
enabled: boolean;
window_start_hour: number;
window_end_hour: number;
default_params: Record<string, unknown>;
};
}
>({
mutationFn: ({ source, payload }) =>
apiFetch<ScheduleConfig>(`/api/v1/admin/scrape/schedules/${source}`, {
method: "PUT",
body: JSON.stringify(payload),
}),
onSuccess: () => {
qc.invalidateQueries({ queryKey: ["scrape-schedules"] });
},
});
}
export function useScraperSettings() {
return useQuery<ScraperSetting[]>({
queryKey: ["scraper-settings"],
queryFn: () =>
apiFetch<{ settings: ScraperSetting[] }>(
"/api/v1/admin/scraper-settings",
).then((r) =>
(r.settings ?? []).map((s) => ({
...s,
request_delay_sec: Number(s.request_delay_sec),
})),
),
staleTime: 30_000,
refetchInterval: 60_000,
});
}
export function useUpdateScraperSetting() {
const qc = useQueryClient();
return useMutation<
ScraperSetting,
Error,
{ source: string; request_delay_sec: number }
>({
mutationFn: ({ source, request_delay_sec }) =>
apiFetch<ScraperSetting>(`/api/v1/admin/scraper-settings/${source}`, {
method: "PUT",
body: JSON.stringify({ source, request_delay_sec }),
}),
onSuccess: () => {
qc.invalidateQueries({ queryKey: ["scraper-settings"] });
},
});
}
// ── Shared Helpers ─────────────────────────────────────────────────────────
export function translateStatus(s: string): string {
switch (s) {
case "running":
return "выполняется";
case "done":
return "готово";
case "failed":
return "ошибка";
case "cancelled":
return "отменено";
case "zombie":
return "зомби";
case "skipped":
return "пропущено";
case "banned":
return "блок";
default:
return s;
}
}
export function formatTime(iso: string | null): string {
if (!iso) return "—";
try {
return new Date(iso).toLocaleString("ru-RU", {
day: "2-digit",
month: "2-digit",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
});
} catch {
return iso;
}
}
export function utcToMsk(hour: number): number {
return (hour + 3) % 24;
}
export function mskToUtc(hour: number): number {
return (hour - 3 + 24) % 24;
}
function formatMskRange(utcStart: number, utcEnd: number): string {
const ms = utcToMsk(utcStart);
const me = utcToMsk(utcEnd);
return `${String(ms).padStart(2, "0")}:00${String(me).padStart(2, "0")}:00 МСК`;
}
// ── ResultPanel (reusable) ─────────────────────────────────────────────────
export interface ResultPanelProps<TData> {
mut: {
data?: TData;
error: Error | null;
isPending: boolean;
isSuccess: boolean;
};
}
export function ResultPanel<TData>({ mut }: ResultPanelProps<TData>) {
if (mut.error) {
return (
<div className="scraper-result scraper-result--error">
<strong>Ошибка:</strong> {mut.error.message}
</div>
);
}
if (mut.isSuccess && mut.data) {
return (
<div className="scraper-result">
<pre>{JSON.stringify(mut.data, null, 2)}</pre>
</div>
);
}
return null;
}
// ── Runs Log Section ───────────────────────────────────────────────────────
interface RunsLogSectionProps {
source: ScraperSource;
}
function RunsLogSection({ source }: RunsLogSectionProps) {
const runsQ = useCitySweepRuns(source);
const cancelMut = useCancelCitySweep(source);
return (
<section className="scraper-section scraper-section--highlight">
<h2>Очередь / Лог прогонов</h2>
<p className="scraper-hint">
Автообновление каждые 5 секунд. Показаны последние 10 прогонов.
</p>
<div className="city-sweep-runs">
<h3>Recent runs (auto-refresh 5s)</h3>
{runsQ.isPending && <p className="scraper-hint">Загрузка</p>}
{runsQ.error && (
<p className="scraper-result scraper-result--error">
Ошибка загрузки runs: {runsQ.error.message}
</p>
)}
{runsQ.data && runsQ.data.length === 0 && (
<p className="scraper-hint">Нет запусков пока.</p>
)}
{runsQ.data && runsQ.data.length > 0 && (
<table className="runs-table">
<thead>
<tr>
<th>#</th>
<th>Статус</th>
<th>Старт</th>
<th>Heartbeat</th>
<th>Anchors</th>
<th>Lots</th>
<th>Houses</th>
<th>Detail</th>
<th>Errors</th>
<th>Действие</th>
</tr>
</thead>
<tbody>
{runsQ.data.map((r) => (
<tr key={r.run_id} className={`run-row run-row--${r.status}`}>
<td>{r.run_id}</td>
<td>
<span className={`run-badge run-badge--${r.status}`}>
{translateStatus(r.status)}
</span>
</td>
<td>{formatTime(r.started_at)}</td>
<td>{formatTime(r.heartbeat_at)}</td>
<td>
{r.counters?.anchors_done ?? 0}/
{r.counters?.anchors_total ?? 0}
</td>
<td>
{r.counters?.lots_fetched ?? 0}{" "}
<span className="run-muted">
(+{r.counters?.lots_inserted ?? 0})
</span>
</td>
<td>
{r.counters?.houses_enriched ?? 0}/
{r.counters?.unique_houses ?? 0}
</td>
<td>
{r.counters?.detail_enriched ?? 0}/
{r.counters?.detail_attempted ?? 0}
</td>
<td>{r.counters?.errors_count ?? 0}</td>
<td>
{r.status === "running" && (
<button
type="button"
className="cancel-btn"
disabled={
cancelMut.isPending &&
cancelMut.variables === r.run_id
}
onClick={() => cancelMut.mutate(r.run_id)}
>
Отменить
</button>
)}
</td>
</tr>
))}
</tbody>
</table>
)}
</div>
</section>
);
}
// ── Cron Schedule Section ──────────────────────────────────────────────────
interface CronScheduleSectionProps {
scheduleSource: ScheduleSourceKey;
paramConfig: SweepParamConfig;
}
function CronScheduleSection({
scheduleSource,
paramConfig,
}: CronScheduleSectionProps) {
const schedulesQ = useSchedules();
const updateScheduleMut = useUpdateSchedule();
const [schEnabled, setSchEnabled] = useState(true);
const [schMskStart, setSchMskStart] = useState(5);
const [schMskEnd, setSchMskEnd] = useState(8);
const [schPages, setSchPages] = useState(
paramConfig.defaults.pages_per_anchor,
);
const [schTopN, setSchTopN] = useState(paramConfig.defaults.detail_top_n ?? 0);
const [schDelay, setSchDelay] = useState(
paramConfig.defaults.request_delay_sec,
);
const [schRadius, setSchRadius] = useState(paramConfig.defaults.radius_m);
const [schEnrichHouses, setSchEnrichHouses] = useState(
paramConfig.defaults.enrich_houses ?? false,
);
const [schEnrichAddress, setSchEnrichAddress] = useState(
paramConfig.defaults.enrich_address ?? false,
);
useEffect(() => {
const sweep = schedulesQ.data?.find((s) => s.source === scheduleSource);
if (!sweep) return;
setSchEnabled(sweep.enabled);
setSchMskStart(utcToMsk(sweep.window_start_hour));
setSchMskEnd(utcToMsk(sweep.window_end_hour));
const p = sweep.default_params ?? {};
if (typeof p.pages_per_anchor === "number")
setSchPages(p.pages_per_anchor);
if (typeof p.detail_top_n === "number") setSchTopN(p.detail_top_n);
if (typeof p.request_delay_sec === "number")
setSchDelay(p.request_delay_sec);
if (typeof p.radius_m === "number") setSchRadius(p.radius_m);
if (typeof p.enrich_houses === "boolean") setSchEnrichHouses(p.enrich_houses);
if (typeof p.enrich_address === "boolean")
setSchEnrichAddress(p.enrich_address);
}, [schedulesQ.data, scheduleSource]);
const currentSweep = schedulesQ.data?.find(
(s) => s.source === scheduleSource,
);
return (
<section className="scraper-section scraper-section--schedule">
<h2>Cron-расписание автозапуска (in-app scheduler)</h2>
<p className="scraper-hint">
Backend периодически (каждую минуту) проверяет это расписание и
запускает city sweep автоматически в случайное время в указанном окне.
Без SSH, без crontab всё в БД и UI.
</p>
{schedulesQ.isPending && <p className="scraper-hint">Загрузка</p>}
{schedulesQ.error && (
<p className="scraper-result scraper-result--error">
Ошибка загрузки: {schedulesQ.error.message}
</p>
)}
{currentSweep && (
<div className="schedule-status">
<div className="schedule-status__row">
<span className="schedule-status__label">Статус:</span>
<span
className={`run-badge run-badge--${currentSweep.enabled ? "running" : "cancelled"}`}
>
{currentSweep.enabled ? "включено" : "отключено"}
</span>
</div>
<div className="schedule-status__row">
<span className="schedule-status__label">Окно (МСК):</span>
<span>
{formatMskRange(
currentSweep.window_start_hour,
currentSweep.window_end_hour,
)}
</span>
<span className="run-muted">
({String(currentSweep.window_start_hour).padStart(2, "0")}:00
{String(currentSweep.window_end_hour).padStart(2, "0")}:00 UTC)
</span>
</div>
<div className="schedule-status__row">
<span className="schedule-status__label">Следующий запуск:</span>
<span>{formatTime(currentSweep.next_run_at)}</span>
</div>
<div className="schedule-status__row">
<span className="schedule-status__label">Последний запуск:</span>
<span>
{currentSweep.last_run_at
? formatTime(currentSweep.last_run_at)
: "—"}
{currentSweep.last_run_id &&
` (run #${currentSweep.last_run_id})`}
</span>
</div>
</div>
)}
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
const defaultParams: Record<string, unknown> = {
pages_per_anchor: schPages,
request_delay_sec: schDelay,
radius_m: schRadius,
};
if (paramConfig.hasDetailParams) {
defaultParams.detail_top_n = schTopN;
defaultParams.enrich_houses = schEnrichHouses;
}
if (paramConfig.hasEnrichAddress) {
defaultParams.enrich_address = schEnrichAddress;
}
updateScheduleMut.mutate({
source: scheduleSource,
payload: {
enabled: schEnabled,
window_start_hour: mskToUtc(schMskStart),
window_end_hour: mskToUtc(schMskEnd),
default_params: defaultParams,
},
});
}}
>
<label className="checkbox">
<input
type="checkbox"
checked={schEnabled}
onChange={(e) => setSchEnabled(e.target.checked)}
/>
Включено
</label>
<label>
Окно начала (МСК)
<input
type="number"
min={0}
max={23}
value={schMskStart}
onChange={(e) => setSchMskStart(parseInt(e.target.value, 10))}
/>
</label>
<label>
Окно окончания (МСК)
<input
type="number"
min={0}
max={23}
value={schMskEnd}
onChange={(e) => setSchMskEnd(parseInt(e.target.value, 10))}
/>
</label>
<label>
Pages/anchor
<input
type="number"
min={1}
max={10}
value={schPages}
onChange={(e) => setSchPages(parseInt(e.target.value, 10))}
/>
</label>
{paramConfig.hasDetailParams && (
<>
<label>
Detail top-N
<input
type="number"
min={0}
max={30}
value={schTopN}
onChange={(e) => setSchTopN(parseInt(e.target.value, 10))}
/>
</label>
</>
)}
<label>
Delay (сек)
<input
type="number"
step={0.5}
min={3}
max={15}
value={schDelay}
onChange={(e) => setSchDelay(parseFloat(e.target.value))}
/>
</label>
<label>
Радиус (м)
<input
type="number"
min={500}
max={5000}
step={100}
value={schRadius}
onChange={(e) => setSchRadius(parseInt(e.target.value, 10))}
/>
</label>
{paramConfig.hasDetailParams && (
<label className="checkbox">
<input
type="checkbox"
checked={schEnrichHouses}
onChange={(e) => setSchEnrichHouses(e.target.checked)}
/>
Enrich houses
</label>
)}
{paramConfig.hasEnrichAddress && (
<label className="checkbox">
<input
type="checkbox"
checked={schEnrichAddress}
onChange={(e) => setSchEnrichAddress(e.target.checked)}
/>
Enrich address
</label>
)}
<button type="submit" disabled={updateScheduleMut.isPending}>
{updateScheduleMut.isPending
? "Сохраняем…"
: "Сохранить расписание"}
</button>
</form>
{updateScheduleMut.error && (
<div className="scraper-result scraper-result--error">
Ошибка: {updateScheduleMut.error.message}
</div>
)}
{updateScheduleMut.isSuccess && updateScheduleMut.data && (
<div className="scraper-result">
Расписание обновлено. Next run:{" "}
{formatTime(updateScheduleMut.data.next_run_at)}
</div>
)}
</section>
);
}
// ── Per-source Delay Section ───────────────────────────────────────────────
interface PerSourceDelaySectionProps {
source: ScraperSource;
delayMin?: number;
delayMax?: number;
}
function PerSourceDelaySection({
source,
delayMin = 1,
delayMax = 60,
}: PerSourceDelaySectionProps) {
const settingsQ = useScraperSettings();
const updateMut = useUpdateScraperSetting();
const currentSetting = settingsQ.data?.find((s) => s.source === source);
const [delayInput, setDelayInput] = useState<string>("5.0");
const [initialized, setInitialized] = useState(false);
useEffect(() => {
if (currentSetting && !initialized) {
setDelayInput(String(currentSetting.request_delay_sec));
setInitialized(true);
}
}, [currentSetting, initialized]);
return (
<section className="scraper-section">
<h2>Задержка запросов (этот скраппер)</h2>
<p className="scraper-hint">
Задержка применяется только к скрапперу <strong>{source}</strong>.
Изменение вступает в силу немедленно без рестарта backend. Глобальный
нижний порог настраивается на{" "}
<a href="/scrapers">странице хаба скрапперов</a>.
</p>
{settingsQ.isPending && (
<p className="scraper-hint">Загрузка настроек</p>
)}
{settingsQ.error && (
<p className="scraper-result scraper-result--error">
Ошибка загрузки: {settingsQ.error.message}
</p>
)}
{currentSetting && (
<div className="schedule-status" style={{ marginBottom: "12px" }}>
<div className="schedule-status__row">
<span className="schedule-status__label">Текущая задержка:</span>
<span>
<strong>{currentSetting.request_delay_sec} сек</strong>
</span>
</div>
{currentSetting.updated_at && (
<div className="schedule-status__row">
<span className="schedule-status__label">Обновлено:</span>
<span>{formatTime(currentSetting.updated_at)}</span>
</div>
)}
{currentSetting.description && (
<div className="schedule-status__row">
<span className="schedule-status__label">Описание:</span>
<span className="run-muted">{currentSetting.description}</span>
</div>
)}
</div>
)}
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
updateMut.mutate({
source,
request_delay_sec: parseFloat(delayInput),
});
}}
>
<label>
Задержка (сек, {delayMin}{delayMax})
<input
type="number"
min={delayMin}
max={delayMax}
step={0.5}
value={delayInput}
onChange={(e) => setDelayInput(e.target.value)}
required
/>
</label>
<button type="submit" disabled={updateMut.isPending}>
{updateMut.isPending ? "Сохраняем…" : "Сохранить задержку"}
</button>
</form>
{updateMut.error && (
<div className="scraper-result scraper-result--error">
<strong>Ошибка:</strong> {updateMut.error.message}
</div>
)}
{updateMut.isSuccess && updateMut.data && (
<div className="scraper-result">
Сохранено. Задержка:{" "}
<strong>
{Number(updateMut.data.request_delay_sec).toFixed(1)} сек
</strong>
</div>
)}
</section>
);
}
// ── City Sweep Section (shared trigger) ───────────────────────────────────
interface CitySweepSectionProps {
source: ScraperSource;
paramConfig: SweepParamConfig;
}
function CitySweepSection({ source, paramConfig }: CitySweepSectionProps) {
const sweepMut = useStartCitySweep(source);
const [sweepPages, setSweepPages] = useState(
String(paramConfig.defaults.pages_per_anchor),
);
const [sweepTopN, setSweepTopN] = useState(
String(paramConfig.defaults.detail_top_n ?? 0),
);
const [sweepDelay, setSweepDelay] = useState(
String(paramConfig.defaults.request_delay_sec),
);
const [sweepRadius, setSweepRadius] = useState(
String(paramConfig.defaults.radius_m),
);
const [sweepEnrichHouses, setSweepEnrichHouses] = useState(
paramConfig.defaults.enrich_houses ?? false,
);
const [sweepEnrichAddress, setSweepEnrichAddress] = useState(
paramConfig.defaults.enrich_address ?? false,
);
return (
<section className="scraper-section">
<h2>Полный обход ЕКБ (city sweep)</h2>
<p className="scraper-hint">
Iterate anchors × N pages save listings enrich. Запускается в
background. Можно отменить в логе прогонов ниже.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
const body: Record<string, unknown> = {
pages_per_anchor: parseInt(sweepPages, 10),
request_delay_sec: parseFloat(sweepDelay),
radius_m: parseInt(sweepRadius, 10),
};
if (paramConfig.hasDetailParams) {
body.detail_top_n = parseInt(sweepTopN, 10);
body.enrich_houses = sweepEnrichHouses;
}
if (paramConfig.hasEnrichAddress) {
body.enrich_address = sweepEnrichAddress;
}
sweepMut.mutate(body);
}}
>
<label>
Страниц на anchor
<input
type="number"
min={1}
max={10}
value={sweepPages}
onChange={(e) => setSweepPages(e.target.value)}
/>
</label>
{paramConfig.hasDetailParams && (
<label>
Detail top-N
<input
type="number"
min={0}
max={30}
value={sweepTopN}
onChange={(e) => setSweepTopN(e.target.value)}
/>
</label>
)}
<label>
Delay (сек)
<input
type="number"
step={0.5}
min={3}
max={15}
value={sweepDelay}
onChange={(e) => setSweepDelay(e.target.value)}
/>
</label>
<label>
Радиус (м)
<input
type="number"
min={500}
max={5000}
step={100}
value={sweepRadius}
onChange={(e) => setSweepRadius(e.target.value)}
/>
</label>
{paramConfig.hasDetailParams && (
<label className="checkbox">
<input
type="checkbox"
checked={sweepEnrichHouses}
onChange={(e) => setSweepEnrichHouses(e.target.checked)}
/>
Enrich houses
</label>
)}
{paramConfig.hasEnrichAddress && (
<label className="checkbox">
<input
type="checkbox"
checked={sweepEnrichAddress}
onChange={(e) => setSweepEnrichAddress(e.target.checked)}
/>
Enrich address
</label>
)}
<button type="submit" disabled={sweepMut.isPending}>
{sweepMut.isPending ? "Запускаем…" : "Запустить sweep"}
</button>
</form>
{sweepMut.error && (
<div className="scraper-result scraper-result--error">
<strong>Ошибка:</strong> {sweepMut.error.message}
</div>
)}
{sweepMut.isSuccess && sweepMut.data && (
<div className="scraper-result">
<pre>{JSON.stringify(sweepMut.data, null, 2)}</pre>
</div>
)}
</section>
);
}
// ── ScraperPage config ─────────────────────────────────────────────────────
export interface ScraperPageConfig {
source: ScraperSource;
activeTab: ActiveTab;
displayName: string;
subtitle: string;
scheduleSource: ScheduleSourceKey;
paramConfig: SweepParamConfig;
delayMin?: number;
delayMax?: number;
/** Extra trigger forms rendered inside the "Запуск" section */
extraTriggers?: React.ReactNode;
/** Extra tabs/sections rendered after the standard 4 sections */
extraSections?: React.ReactNode;
}
// ── ScraperPage (main export) ──────────────────────────────────────────────
export function ScraperPage({ config }: { config: ScraperPageConfig }) {
const {
source,
activeTab,
displayName,
subtitle,
scheduleSource,
paramConfig,
delayMin,
delayMax,
extraTriggers,
extraSections,
} = config;
return (
<>
<Topbar active={activeTab} />
<main className="page scraper-page">
<h1 className="scraper-h1">{displayName} admin триггер</h1>
<p className="scraper-subtitle">{subtitle}</p>
{/* Section 1: Запуск — city sweep + extra provider triggers */}
<section className="scraper-section">
<h2>Запуск</h2>
{extraTriggers}
<CitySweepSection source={source} paramConfig={paramConfig} />
</section>
{/* Section 2: Runs log */}
<RunsLogSection source={source} />
{/* Section 3: Cron schedule */}
<CronScheduleSection
scheduleSource={scheduleSource}
paramConfig={paramConfig}
/>
{/* Section 4: Per-source delay */}
<PerSourceDelaySection
source={source}
delayMin={delayMin}
delayMax={delayMax}
/>
{/* Extra sections (e.g. cookies tab for cian) */}
{extraSections}
</main>
</>
);
}

File diff suppressed because it is too large Load diff

View file

@ -1,351 +1,21 @@
"use client";
import { useState } from "react";
import { useMutation, useQuery } from "@tanstack/react-query";
import { useEffect } from "react";
import { useRouter } from "next/navigation";
import "@/components/trade-in/trade-in.css";
import { Topbar } from "@/components/trade-in/Topbar";
import { apiFetch } from "@/lib/api";
interface UploadResponse {
ok?: boolean;
userId?: number;
cookieCount?: number;
detail?: string;
}
interface TestAuthResponse {
authenticated: boolean;
userId?: number;
reason?: string;
}
interface CookieEditorEntry {
name: string;
value: string | number | boolean;
[key: string]: unknown;
}
function convertCookiesFormat(input: string): Record<string, string> {
const parsed: unknown = JSON.parse(input.trim());
if (Array.isArray(parsed)) {
// Cookie-Editor array format: [{name, value, domain, ...}, ...]
return (parsed as CookieEditorEntry[]).reduce<Record<string, string>>(
(acc, c) => {
if (
typeof c === "object" &&
c !== null &&
"name" in c &&
"value" in c
) {
acc[String(c.name)] = String(c.value);
}
return acc;
},
{},
);
}
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
// Already a dict: {"name1": "val1", "name2": "val2"}
return Object.fromEntries(
Object.entries(parsed as Record<string, unknown>).map(([k, v]) => [
k,
String(v),
]),
);
}
throw new Error("Expected JSON array or plain object");
}
function parsePreview(raw: string): {
cookies: Record<string, string> | null;
error: string | null;
} {
if (!raw.trim()) return { cookies: null, error: null };
try {
const cookies = convertCookiesFormat(raw);
return { cookies, error: null };
} catch (e) {
return { cookies: null, error: e instanceof Error ? e.message : String(e) };
}
}
export default function CianCookiesPage() {
const [rawInput, setRawInput] = useState("");
const [showInstructions, setShowInstructions] = useState(false);
const preview = parsePreview(rawInput);
const previewKeys = preview.cookies ? Object.keys(preview.cookies) : [];
const uploadMutation = useMutation({
mutationFn: () => {
if (!preview.cookies) {
throw new Error("Invalid JSON — fix the input first");
}
return apiFetch<UploadResponse>(
"/api/v1/admin/scrape/cian/upload-cookies",
{
method: "POST",
body: JSON.stringify(preview.cookies),
},
);
},
onSuccess: () => setRawInput(""),
});
const testAuthQuery = useQuery({
queryKey: ["cian-test-auth"],
queryFn: () =>
apiFetch<TestAuthResponse>("/api/v1/admin/scrape/cian/test-auth"),
enabled: false, // only on explicit button click
retry: false,
});
const canUpload =
!!preview.cookies && previewKeys.length > 0 && !uploadMutation.isPending;
/**
* /scrapers/cian-cookies redirects to /scrapers/cian where cookie management
* is now integrated as a dedicated section at the bottom of the Cian page.
*/
export default function CianCookiesRedirect() {
const router = useRouter();
useEffect(() => {
router.replace("/scrapers/cian");
}, [router]);
return (
<>
<Topbar active="cian-cookies" />
<main className="page scraper-page">
<h1 className="scraper-h1">Cian обновление cookies</h1>
<p className="scraper-subtitle">
Позволяет обновить сессию Cian для скрапера оценщика без перезапуска
backend. Endpoints:{" "}
<code>POST /api/v1/admin/scrape/cian/upload-cookies</code> ·{" "}
<code>GET /api/v1/admin/scrape/cian/test-auth</code>
</p>
{/* Instructions collapsible */}
<div className="scraper-section">
<button
onClick={() => setShowInstructions((v) => !v)}
style={{
background: "none",
border: "none",
padding: 0,
cursor: "pointer",
fontSize: 14,
fontWeight: 600,
color: "var(--accent, #1d4ed8)",
display: "flex",
alignItems: "center",
gap: 6,
}}
>
<span>{showInstructions ? "▼" : "▶"}</span>
Как получить cookies (инструкция)
</button>
{showInstructions && (
<ol
style={{
marginTop: 12,
paddingLeft: 20,
fontSize: 13,
color: "var(--fg, #374151)",
lineHeight: 1.7,
}}
>
<li>
Открой{" "}
<a
href="https://www.cian.ru/"
target="_blank"
rel="noopener noreferrer"
>
cian.ru
</a>{" "}
и войди под нужным аккаунтом.
</li>
<li>
Установи расширение <strong>Cookie-Editor</strong>{" "}
(Chrome/Firefox) <em>или</em> открой DevTools Application
Cookies www.cian.ru.
</li>
<li>
<strong>Cookie-Editor:</strong> нажми Export Export as JSON
скопируй массив{" "}
<code>[{"{"}name, value, ...{"}"}]</code>.
</li>
<li>
<strong>DevTools (ручной способ):</strong> собери объект{" "}
<code>
{"{"}
{'"'}name1{'"'}: {'"'}val1{'"'}, ...{"}"}
</code>
.
</li>
<li>
Вставь JSON в поле ниже формат определится автоматически.
</li>
<li>
Убедись что preview показывает нужные ключи, затем нажми
Upload.
</li>
</ol>
)}
</div>
{/* Cookie input */}
<div className="scraper-section">
<h2>Cookie JSON</h2>
<p className="scraper-hint">
Вставьте Cookie-Editor array или plain dict. Формат определяется
автоматически.
</p>
<div className="scraper-form">
<label className="full">
<textarea
value={rawInput}
onChange={(e) => {
setRawInput(e.target.value);
uploadMutation.reset();
}}
rows={15}
placeholder={`Формат 1 — Cookie-Editor array:\n[\n {"name": "session_key", "value": "abc123", "domain": ".cian.ru"},\n {"name": "_CSRF", "value": "xyz", "domain": ".cian.ru"}\n]\n\ормат 2 — dict (DevTools manual):\n{\n "session_key": "abc123",\n "_CSRF": "xyz"\n}`}
style={{
fontFamily: "var(--font-mono, ui-monospace, monospace)",
fontSize: 12,
resize: "vertical",
minHeight: 240,
padding: "8px 10px",
border: "1px solid var(--border-strong, #d1d5db)",
borderRadius: 6,
background: "#fff",
color: "var(--fg, #111)",
width: "100%",
}}
/>
</label>
{/* Preview */}
{rawInput.trim() && (
<div
className={`scraper-result${preview.error ? " scraper-result--error" : ""}`}
style={{ fontFamily: "inherit" }}
>
{preview.error ? (
<span>JSON error: {preview.error}</span>
) : (
<span style={{ color: "var(--success, #0a7a3a)" }}>
Detected <strong>{previewKeys.length} cookies</strong>:{" "}
{previewKeys.slice(0, 8).join(", ")}
{previewKeys.length > 8
? ` … +${previewKeys.length - 8} more`
: ""}
</span>
)}
</div>
)}
<button
onClick={(e) => {
e.preventDefault();
uploadMutation.mutate();
}}
disabled={!canUpload}
title={!preview.cookies ? "Вставьте корректный JSON" : undefined}
>
{uploadMutation.isPending ? "Uploading…" : "Upload Cookies"}
</button>
<button
onClick={(e) => {
e.preventDefault();
void testAuthQuery.refetch();
}}
disabled={testAuthQuery.isFetching}
style={{
background: "var(--surface-2, #f3f4f6)",
color: "var(--fg, #1f2937)",
border: "1px solid var(--border-strong, #d1d5db)",
justifySelf: "start",
}}
>
{testAuthQuery.isFetching ? "Checking…" : "Test Current Session"}
</button>
</div>
{/* Upload result */}
{uploadMutation.isSuccess && (
<div
className="scraper-result"
style={{
fontFamily: "inherit",
background: uploadMutation.data.ok
? "var(--success-soft, #f0fdf4)"
: "var(--danger-soft, #fef2f2)",
borderColor: uploadMutation.data.ok
? "var(--success, #0a7a3a)"
: "var(--danger, #b3261e)",
color: uploadMutation.data.ok
? "var(--success, #0a7a3a)"
: "var(--danger, #b3261e)",
}}
>
{uploadMutation.data.ok ? (
<>
<strong>Cookies uploaded.</strong> userId={" "}
{uploadMutation.data.userId ?? "—"} · cookieCount={" "}
{uploadMutation.data.cookieCount ?? previewKeys.length}
</>
) : (
<>
<strong>Upload failed:</strong>{" "}
{uploadMutation.data.detail ?? "Unknown error"}
</>
)}
</div>
)}
{uploadMutation.isError && (
<div className="scraper-result scraper-result--error">
<strong>Error:</strong> {String(uploadMutation.error)}
</div>
)}
{/* Test auth result */}
{(testAuthQuery.isSuccess || testAuthQuery.isError) && (
<div
className="scraper-result"
style={{
fontFamily: "inherit",
background:
testAuthQuery.isSuccess && testAuthQuery.data.authenticated
? "var(--success-soft, #f0fdf4)"
: "var(--danger-soft, #fef2f2)",
borderColor:
testAuthQuery.isSuccess && testAuthQuery.data.authenticated
? "var(--success, #0a7a3a)"
: "var(--danger, #b3261e)",
color:
testAuthQuery.isSuccess && testAuthQuery.data.authenticated
? "var(--success, #0a7a3a)"
: "var(--danger, #b3261e)",
}}
>
{testAuthQuery.isError ? (
<>
<strong>Test auth error:</strong>{" "}
{String(testAuthQuery.error)}
</>
) : testAuthQuery.data.authenticated ? (
<>
<strong>Authenticated.</strong> userId={" "}
{testAuthQuery.data.userId ?? "—"}
</>
) : (
<>
<strong>Not authenticated.</strong>{" "}
{testAuthQuery.data.reason
? `Reason: ${testAuthQuery.data.reason}`
: "Session expired or cookies invalid."}
</>
)}
</div>
)}
</div>
</main>
</>
<p style={{ padding: "2rem", fontFamily: "sans-serif" }}>
Перенаправление на страницу Cian скраппера
</p>
);
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,423 @@
"use client";
import { useEffect, useState } from "react";
import "@/components/trade-in/trade-in.css";
import { Topbar } from "@/components/trade-in/Topbar";
import { API_BASE_URL } from "@/lib/api";
import {
formatTime,
translateStatus,
useScraperSettings,
useSchedules,
useUpdateScraperSetting,
useUpdateSchedule,
utcToMsk,
type ScheduleConfig,
type ScheduleSourceKey,
} from "@/app/scrapers/_components/ScraperPage";
// ── Global delay section ───────────────────────────────────────────────────
function GlobalDelaySetting() {
const settingsQ = useScraperSettings();
const updateMut = useUpdateScraperSetting();
const globalSetting = settingsQ.data?.find((s) => s.source === "global");
const [delayInput, setDelayInput] = useState("0");
const [initialized, setInitialized] = useState(false);
useEffect(() => {
if (globalSetting && !initialized) {
setDelayInput(String(globalSetting.request_delay_sec));
setInitialized(true);
}
}, [globalSetting, initialized]);
return (
<section className="scraper-section scraper-section--highlight">
<h2>Глобальная задержка запросов (нижний порог)</h2>
<p className="scraper-hint">
Глобальный делэй нижний порог для всех скрапперов (max с per-source
defaults). Если 0 только per-source defaults. Изменение вступает в
силу немедленно без перезапуска процесса. Для per-source задержек
зайдите на страницу конкретного скраппера.
</p>
{settingsQ.isPending && (
<p className="scraper-hint">Загрузка настроек</p>
)}
{settingsQ.error && (
<p className="scraper-result scraper-result--error">
Ошибка загрузки: {settingsQ.error.message}
</p>
)}
{globalSetting && (
<div className="schedule-status" style={{ marginBottom: "12px" }}>
<div className="schedule-status__row">
<span className="schedule-status__label">Текущий global delay:</span>
<span>
<strong>{globalSetting.request_delay_sec} сек</strong>
</span>
</div>
{globalSetting.updated_at && (
<div className="schedule-status__row">
<span className="schedule-status__label">Обновлено:</span>
<span>{formatTime(globalSetting.updated_at)}</span>
</div>
)}
</div>
)}
{/* Per-source read-only table */}
{settingsQ.data && settingsQ.data.length > 0 && (
<div style={{ marginBottom: "16px" }}>
<p
style={{
fontSize: "0.85rem",
fontWeight: 600,
marginBottom: "6px",
}}
>
Все per-source задержки (read-only здесь, редактировать на
страницах провайдеров):
</p>
<table className="runs-table">
<thead>
<tr>
<th>Source</th>
<th>Delay (сек)</th>
<th>Описание</th>
<th>Обновлено</th>
</tr>
</thead>
<tbody>
{settingsQ.data.map((s) => (
<tr
key={s.source}
className={
s.source === "global"
? "run-row run-row--running"
: "run-row"
}
>
<td>
<code>{s.source}</code>
</td>
<td>{s.request_delay_sec.toFixed(1)}</td>
<td
style={{
fontSize: "0.8rem",
color: "var(--muted, #5b6066)",
}}
>
{s.description ?? "—"}
</td>
<td style={{ fontSize: "0.8rem" }}>
{formatTime(s.updated_at)}
</td>
</tr>
))}
</tbody>
</table>
</div>
)}
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
updateMut.mutate({
source: "global",
request_delay_sec: parseFloat(delayInput),
});
}}
>
<label>
Global delay (сек, 030)
<input
type="number"
step={0.5}
min={0}
max={30}
value={delayInput}
onChange={(e) => setDelayInput(e.target.value)}
/>
</label>
<input
type="range"
min={0}
max={30}
step={0.5}
value={delayInput}
onChange={(e) => setDelayInput(e.target.value)}
style={{ width: "100%", gridColumn: "1 / -1" }}
/>
<button
type="submit"
disabled={updateMut.isPending}
style={{ gridColumn: "1 / -1" }}
>
{updateMut.isPending ? "Сохраняем…" : "Сохранить global delay"}
</button>
</form>
{updateMut.error && (
<div className="scraper-result scraper-result--error">
Ошибка: {updateMut.error.message}
</div>
)}
{updateMut.isSuccess && updateMut.data && (
<div className="scraper-result">
Global delay обновлён:{" "}
<strong>
{Number(updateMut.data.request_delay_sec).toFixed(1)} сек
</strong>{" "}
(обновлено {formatTime(updateMut.data.updated_at)})
</div>
)}
</section>
);
}
// ── Schedules summary table ────────────────────────────────────────────────
const SCHEDULE_SOURCE_LABELS: Record<string, string> = {
avito_city_sweep: "Avito city sweep",
cian_city_sweep: "Cian city sweep",
yandex_city_sweep: "Yandex city sweep",
};
const SCHEDULE_SOURCES: ScheduleSourceKey[] = [
"avito_city_sweep",
"cian_city_sweep",
"yandex_city_sweep",
];
function SchedulesSummary() {
const schedulesQ = useSchedules();
const updateMut = useUpdateSchedule();
function handleToggle(schedule: ScheduleConfig) {
updateMut.mutate({
source: schedule.source as ScheduleSourceKey,
payload: {
enabled: !schedule.enabled,
window_start_hour: schedule.window_start_hour,
window_end_hour: schedule.window_end_hour,
default_params: schedule.default_params as Record<string, unknown>,
},
});
}
// Build ordered rows from known sources + any extras
const knownRows = SCHEDULE_SOURCES.map((src) =>
schedulesQ.data?.find((s) => s.source === src),
).filter((s): s is ScheduleConfig => s !== undefined);
const extraRows =
schedulesQ.data?.filter(
(s) => !SCHEDULE_SOURCES.includes(s.source as ScheduleSourceKey),
) ?? [];
const rows = [...knownRows, ...extraRows];
return (
<section className="scraper-section">
<h2>Сводная таблица расписаний</h2>
<p className="scraper-hint">
Быстрое включение/отключение всех cron-расписаний. Параметры и окна
настраиваются на страницах провайдеров. Автообновление каждые 30 сек.
</p>
{schedulesQ.isPending && <p className="scraper-hint">Загрузка</p>}
{schedulesQ.error && (
<p className="scraper-result scraper-result--error">
Ошибка загрузки: {schedulesQ.error.message}
</p>
)}
{rows.length > 0 && (
<table className="runs-table">
<thead>
<tr>
<th>Source</th>
<th>Статус</th>
<th>Окно (МСК)</th>
<th>Следующий запуск</th>
<th>Последний запуск</th>
<th>Последний run</th>
<th>Вкл/выкл</th>
</tr>
</thead>
<tbody>
{rows.map((s) => {
const mskStart = utcToMsk(s.window_start_hour);
const mskEnd = utcToMsk(s.window_end_hour);
const label =
SCHEDULE_SOURCE_LABELS[s.source] ?? s.source;
return (
<tr key={s.source} className="run-row">
<td>
<strong>{label}</strong>
</td>
<td>
<span
className={`run-badge run-badge--${s.enabled ? "running" : "cancelled"}`}
>
{s.enabled ? "включено" : "отключено"}
</span>
</td>
<td>
{String(mskStart).padStart(2, "0")}:00
{String(mskEnd).padStart(2, "0")}:00 МСК
</td>
<td>{formatTime(s.next_run_at)}</td>
<td>{formatTime(s.last_run_at)}</td>
<td>
{s.last_run_id ? (
<span className="run-muted">#{s.last_run_id}</span>
) : (
"—"
)}
</td>
<td>
<button
type="button"
className="cancel-btn"
disabled={updateMut.isPending}
onClick={() => handleToggle(s)}
style={
s.enabled
? undefined
: { background: "var(--success, #27ae60)" }
}
>
{s.enabled ? "Отключить" : "Включить"}
</button>
</td>
</tr>
);
})}
</tbody>
</table>
)}
{rows.length === 0 && !schedulesQ.isPending && !schedulesQ.error && (
<p className="scraper-hint">Расписания не найдены. Они создаются автоматически при первом сохранении на странице провайдера.</p>
)}
{updateMut.error && (
<div className="scraper-result scraper-result--error">
Ошибка обновления: {updateMut.error.message}
</div>
)}
{updateMut.isSuccess && (
<div className="scraper-result">
Расписание {translateStatus(
updateMut.data.enabled ? "running" : "cancelled",
)}.
</div>
)}
</section>
);
}
// ── Provider cards ─────────────────────────────────────────────────────────
const PROVIDERS = [
{
key: "avito",
name: "Avito",
href: "/scrapers/avito",
description:
"City sweep, houses catalog, detail enrichment, IMV evaluation",
},
{
key: "cian",
name: "Cian",
href: "/scrapers/cian",
description: "City sweep, detail enrichment, новостройки ЖК, cookies",
},
{
key: "yandex",
name: "Yandex Realty",
href: "/scrapers/yandex",
description: "City sweep, detail, ЖК landing, valuation",
},
];
function ProviderCards() {
return (
<section className="scraper-section">
<h2>Провайдеры</h2>
<div
style={{
display: "grid",
gridTemplateColumns: "repeat(auto-fit, minmax(240px, 1fr))",
gap: "16px",
marginTop: "12px",
}}
>
{PROVIDERS.map((p) => (
<a
key={p.key}
href={`${API_BASE_URL}${p.href}`}
style={{
display: "block",
padding: "16px 20px",
border: "1px solid var(--border, #e5e7eb)",
borderRadius: 8,
background: "var(--surface, #fff)",
textDecoration: "none",
color: "inherit",
transition: "border-color 0.15s, box-shadow 0.15s",
}}
>
<p
style={{
margin: "0 0 4px",
fontWeight: 700,
fontSize: "1rem",
color: "var(--accent, #1d4ed8)",
}}
>
{p.name}
</p>
<p
style={{
margin: 0,
fontSize: "0.85rem",
color: "var(--muted, #5b6066)",
}}
>
{p.description}
</p>
</a>
))}
</div>
</section>
);
}
// ── Page ───────────────────────────────────────────────────────────────────
export default function ScrapersHubPage() {
return (
<>
<Topbar active="scrapers" />
<main className="page scraper-page">
<h1 className="scraper-h1">Скрапперы хаб</h1>
<p className="scraper-subtitle">
Центр управления всеми скрапперами: глобальные настройки задержки,
сводная таблица расписаний, ссылки на admin-триггеры провайдеров.
</p>
<ProviderCards />
<GlobalDelaySetting />
<SchedulesSummary />
</main>
</>
);
}

View file

@ -1,34 +1,16 @@
"use client";
import { useEffect, useRef, useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { useState } from "react";
import { useMutation } from "@tanstack/react-query";
import "@/components/trade-in/trade-in.css";
import { Topbar } from "@/components/trade-in/Topbar";
import { apiFetch } from "@/lib/api";
import {
ResultPanel,
ScraperPage,
type ScraperPageConfig,
} from "@/app/scrapers/_components/ScraperPage";
// ── Types ──────────────────────────────────────────────────────────────────
interface ScraperSetting {
source: string;
request_delay_sec: number;
description: string | null;
updated_at: string | null;
}
interface ScraperSettingUpdate {
source: string;
request_delay_sec: number;
}
interface ScrapeAroundInput {
lat: number;
lon: number;
radius_m: number;
sources: string[];
multi_room_yandex?: boolean;
deep_yandex?: boolean;
}
// ── Yandex-specific types + hooks ──────────────────────────────────────────
interface ScrapeAroundResp {
total_fetched: number;
@ -75,43 +57,19 @@ interface YandexValuationTriggerResp {
history_items_count: number;
}
// ── Hooks ──────────────────────────────────────────────────────────────────
function useScraperSettings() {
return useQuery<ScraperSetting[]>({
queryKey: ["scraper-settings"],
queryFn: () =>
apiFetch<{ settings: ScraperSetting[] }>("/api/v1/admin/scraper-settings").then(
(r) =>
(r.settings ?? []).map((s) => ({
...s,
request_delay_sec: Number(s.request_delay_sec),
})),
),
refetchInterval: 30_000,
});
}
function useUpdateScraperSetting() {
const qc = useQueryClient();
return useMutation<
ScraperSetting,
Error,
{ source: string; payload: Omit<ScraperSettingUpdate, "source"> }
>({
mutationFn: ({ source, payload }) =>
apiFetch<ScraperSetting>(`/api/v1/admin/scraper-settings/${source}`, {
method: "PUT",
body: JSON.stringify({ source, ...payload }),
}),
onSuccess: () => {
qc.invalidateQueries({ queryKey: ["scraper-settings"] });
},
});
}
function useScrapeAround() {
return useMutation<ScrapeAroundResp, Error, ScrapeAroundInput>({
return useMutation<
ScrapeAroundResp,
Error,
{
lat: number;
lon: number;
radius_m: number;
sources: string[];
multi_room_yandex?: boolean;
deep_yandex?: boolean;
}
>({
mutationFn: (input) =>
apiFetch<ScrapeAroundResp>("/api/v1/admin/scrape", {
method: "POST",
@ -150,7 +108,12 @@ function useScrapeYandexValuation() {
return useMutation<
YandexValuationTriggerResp,
Error,
{ address: string; offer_category: string; offer_type: string; page: number }
{
address: string;
offer_category: string;
offer_type: string;
page: number;
}
>({
mutationFn: ({ address, offer_category, offer_type, page }) => {
const qs = new URLSearchParams({
@ -167,69 +130,9 @@ function useScrapeYandexValuation() {
});
}
// ── Result panel ───────────────────────────────────────────────────────────
interface ResultPanelProps<TData> {
mut: {
data?: TData;
error: Error | null;
isPending: boolean;
isSuccess: boolean;
};
}
function ResultPanel<TData>({ mut }: ResultPanelProps<TData>) {
if (mut.error) {
return (
<div className="scraper-result scraper-result--error">
<strong>Ошибка:</strong> {mut.error.message}
</div>
);
}
if (mut.isSuccess && mut.data) {
return (
<div className="scraper-result">
<pre>{JSON.stringify(mut.data, null, 2)}</pre>
</div>
);
}
return null;
}
// ── Helpers ────────────────────────────────────────────────────────────────
function formatTime(iso: string | null): string {
if (!iso) return "—";
try {
return new Date(iso).toLocaleString("ru-RU", {
day: "2-digit",
month: "2-digit",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
});
} catch {
return iso;
}
}
// ── Page ───────────────────────────────────────────────────────────────────
export default function YandexScraperPage() {
// Settings
const settingsQ = useScraperSettings();
const updateSettingMut = useUpdateScraperSetting();
const [delayInput, setDelayInput] = useState<string>("5.0");
const delayInitializedRef = useRef(false);
useEffect(() => {
const yandex = settingsQ.data?.find((s) => s.source === "yandex");
if (yandex && !delayInitializedRef.current) {
setDelayInput(String(yandex.request_delay_sec));
delayInitializedRef.current = true;
}
}, [settingsQ.data]);
// ── Yandex extra triggers ──────────────────────────────────────────────────
function YandexExtraTriggers() {
// Around
const [lat, setLat] = useState("56.8400");
const [lon, setLon] = useState("60.6050");
@ -259,317 +162,263 @@ export default function YandexScraperPage() {
const [valPage, setValPage] = useState("1");
const valMut = useScrapeYandexValuation();
const yandexSetting = settingsQ.data?.find((s) => s.source === "yandex");
return (
<>
<Topbar active="yandex" />
<main className="page scraper-page">
<h1 className="scraper-h1">Yandex скрапер admin триггер</h1>
<p className="scraper-subtitle">
Ручной запуск sub-pipeline&apos;ов YandexRealtyScraper v1 + глобальная настройка
задержки между запросами.
{/* 1. Around point */}
<div className="scraper-section">
<h3>1. Search around (общий /admin/scrape)</h3>
<p className="scraper-hint">
Yandex SERP path-based vtorichka lat/lon/radius логируется, scraper
берёт{" "}
<code>/&#123;ekaterinburg&#125;/kupit/kvartira/vtorichniy-rynok/</code>.
multi-room split не реализован для DOM-парсера. Deep пробует pages ×
sorts комбинации.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
aroundMut.mutate({
lat: parseFloat(lat),
lon: parseFloat(lon),
radius_m: parseInt(radius, 10),
sources: ["yandex"],
multi_room_yandex: multiRoom,
deep_yandex: deep,
});
}}
>
<label>
Lat
<input
value={lat}
onChange={(e) => setLat(e.target.value)}
required
/>
</label>
<label>
Lon
<input
value={lon}
onChange={(e) => setLon(e.target.value)}
required
/>
</label>
<label>
Radius (м)
<input
type="number"
min={100}
max={10000}
step={100}
value={radius}
onChange={(e) => setRadius(e.target.value)}
required
/>
</label>
<label className="checkbox">
<input
type="checkbox"
checked={multiRoom}
onChange={(e) => setMultiRoom(e.target.checked)}
/>
multi_room_yandex (5 rooms split)
</label>
<label className="checkbox">
<input
type="checkbox"
checked={deep}
onChange={(e) => setDeep(e.target.checked)}
/>
deep_yandex (5 × 3 × 2 = 30 requests)
</label>
<button type="submit" disabled={aroundMut.isPending}>
{aroundMut.isPending ? "Скрейпим…" : "Запустить"}
</button>
</form>
<ResultPanel mut={aroundMut} />
</div>
{/* 0. Global delay */}
<section className="scraper-section scraper-section--highlight">
<h2>Глобальная задержка запросов</h2>
<p className="scraper-hint">
Настройка применяется ко ВСЕМ Yandex-скрейперам (SERP, detail, ЖК, valuation)
при создании следующего инстанса (in-process cache 60s). Меняется без рестарта
backend.
</p>
{/* 2. Detail */}
<div className="scraper-section">
<h3>2. Detail page enrichment</h3>
<p className="scraper-hint">
Product JSON-LD price (exact int) + DOM-секции
description/agency/metro/photos.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
detailMut.mutate({ offer_url: detailUrl });
}}
>
<label className="full">
offer_url
<input
value={detailUrl}
onChange={(e) => setDetailUrl(e.target.value)}
required
placeholder="https://realty.yandex.ru/offer/..."
/>
</label>
<button type="submit" disabled={detailMut.isPending}>
{detailMut.isPending ? "Парсим…" : "Запустить"}
</button>
</form>
<ResultPanel mut={detailMut} />
</div>
{settingsQ.isPending && <p className="scraper-hint">Загрузка настроек</p>}
{settingsQ.error && (
<p className="scraper-result scraper-result--error">
Ошибка загрузки настроек: {settingsQ.error.message}
</p>
)}
{/* 3. Newbuilding */}
<div className="scraper-section">
<h3>3. ЖК (Newbuilding) landing</h3>
<p className="scraper-hint">
URL /&#123;city&#125;/kupit/novostrojka/&#123;slug&#125;-&#123;id&#125;/ извлекает coords
inline, rating, text_reviews_count, developer_name.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
nbMut.mutate({ slug: nbSlug, id: nbId, city: nbCity });
}}
>
<label>
slug
<input
value={nbSlug}
onChange={(e) => setNbSlug(e.target.value)}
required
placeholder="tatlin"
/>
</label>
<label>
id
<input
value={nbId}
onChange={(e) => setNbId(e.target.value)}
required
placeholder="1592987"
/>
</label>
<label>
city
<input
value={nbCity}
onChange={(e) => setNbCity(e.target.value)}
required
placeholder="ekaterinburg"
/>
</label>
<button type="submit" disabled={nbMut.isPending}>
{nbMut.isPending ? "Парсим…" : "Запустить"}
</button>
</form>
<ResultPanel mut={nbMut} />
</div>
{yandexSetting && (
<div className="schedule-status" style={{ marginBottom: "12px" }}>
<div className="schedule-status__row">
<span className="schedule-status__label">Текущая задержка:</span>
<span>
<strong>{yandexSetting.request_delay_sec} сек</strong>
</span>
</div>
{yandexSetting.updated_at && (
<div className="schedule-status__row">
<span className="schedule-status__label">Обновлено:</span>
<span>{formatTime(yandexSetting.updated_at)}</span>
</div>
)}
{yandexSetting.description && (
<div className="schedule-status__row">
<span className="schedule-status__label">Описание:</span>
<span className="run-muted">{yandexSetting.description}</span>
</div>
)}
</div>
)}
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
updateSettingMut.mutate({
source: "yandex",
payload: { request_delay_sec: parseFloat(delayInput) },
});
}}
>
<label>
Задержка (сек, 160)
<input
type="number"
min={1}
max={60}
step={0.5}
value={delayInput}
onChange={(e) => setDelayInput(e.target.value)}
required
/>
</label>
<button type="submit" disabled={updateSettingMut.isPending}>
{updateSettingMut.isPending ? "Сохраняем…" : "Сохранить"}
</button>
</form>
{updateSettingMut.error && (
<div className="scraper-result scraper-result--error">
<strong>Ошибка:</strong> {updateSettingMut.error.message}
</div>
)}
{updateSettingMut.isSuccess && updateSettingMut.data && (
<div className="scraper-result">
Сохранено. Новая задержка:{" "}
<strong>{updateSettingMut.data.request_delay_sec} сек</strong>
</div>
)}
</section>
{/* 1. Around point */}
<section className="scraper-section">
<h2>1. Search around (общий /admin/scrape)</h2>
<p className="scraper-hint">
Yandex SERP path-based vtorichka lat/lon/radius логируется, scraper берёт{" "}
<code>/&#123;ekaterinburg&#125;/kupit/kvartira/vtorichniy-rynok/</code>.
multi-room split не реализован для DOM-парсера (запросы будут одинаковыми).
Deep пробует pages × sorts комбинации.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
aroundMut.mutate({
lat: parseFloat(lat),
lon: parseFloat(lon),
radius_m: parseInt(radius, 10),
sources: ["yandex"],
multi_room_yandex: multiRoom,
deep_yandex: deep,
});
}}
>
<label>
Lat
<input
value={lat}
onChange={(e) => setLat(e.target.value)}
required
/>
</label>
<label>
Lon
<input
value={lon}
onChange={(e) => setLon(e.target.value)}
required
/>
</label>
<label>
Radius (м)
<input
type="number"
min={100}
max={10000}
step={100}
value={radius}
onChange={(e) => setRadius(e.target.value)}
required
/>
</label>
<label className="checkbox">
<input
type="checkbox"
checked={multiRoom}
onChange={(e) => setMultiRoom(e.target.checked)}
/>
multi_room_yandex (5 rooms split)
</label>
<label className="checkbox">
<input
type="checkbox"
checked={deep}
onChange={(e) => setDeep(e.target.checked)}
/>
deep_yandex (5 × 3 × 2 = 30 requests)
</label>
<button type="submit" disabled={aroundMut.isPending}>
{aroundMut.isPending ? "Скрейпим…" : "Запустить"}
</button>
</form>
<ResultPanel mut={aroundMut} />
</section>
{/* 2. Detail */}
<section className="scraper-section">
<h2>2. Detail page enrichment</h2>
<p className="scraper-hint">
Product JSON-LD price (exact int) + DOM-секции description/agency/metro/photos.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
detailMut.mutate({ offer_url: detailUrl });
}}
>
<label className="full">
offer_url
<input
value={detailUrl}
onChange={(e) => setDetailUrl(e.target.value)}
required
placeholder="https://realty.yandex.ru/offer/..."
/>
</label>
<button type="submit" disabled={detailMut.isPending}>
{detailMut.isPending ? "Парсим…" : "Запустить"}
</button>
</form>
<ResultPanel mut={detailMut} />
</section>
{/* 3. ЖК Newbuilding */}
<section className="scraper-section">
<h2>3. ЖК (Newbuilding) landing</h2>
<p className="scraper-hint">
URL /&#123;city&#125;/kupit/novostrojka/&#123;slug&#125;-&#123;id&#125;/ извлекает coords inline,
rating, text_reviews_count, developer_name.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
nbMut.mutate({ slug: nbSlug, id: nbId, city: nbCity });
}}
>
<label>
slug
<input
value={nbSlug}
onChange={(e) => setNbSlug(e.target.value)}
required
placeholder="tatlin"
/>
</label>
<label>
id
<input
value={nbId}
onChange={(e) => setNbId(e.target.value)}
required
placeholder="1592987"
/>
</label>
<label>
city
<input
value={nbCity}
onChange={(e) => setNbCity(e.target.value)}
required
placeholder="ekaterinburg"
/>
</label>
<button type="submit" disabled={nbMut.isPending}>
{nbMut.isPending ? "Парсим…" : "Запустить"}
</button>
</form>
<ResultPanel mut={nbMut} />
</section>
{/* 4. Valuation */}
<section className="scraper-section">
<h2>4. Valuation (anonymous history)</h2>
<p className="scraper-hint">
Anonymous GET /otsenka-kvartiry-po-adresu-onlayn/ house meta + до 30 history
items на page.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
valMut.mutate({
address: valAddress,
offer_category: valCategory,
offer_type: valType,
page: parseInt(valPage, 10),
});
}}
>
<label className="full">
Адрес
<input
value={valAddress}
onChange={(e) => setValAddress(e.target.value)}
required
placeholder="Свердловская область, Екатеринбург, улица Учителей, 18"
/>
</label>
<label>
offer_category
<select
value={valCategory}
onChange={(e) => setValCategory(e.target.value)}
>
<option value="APARTMENT">APARTMENT</option>
<option value="HOUSE">HOUSE</option>
<option value="GARAGE">GARAGE</option>
<option value="COMMERCIAL">COMMERCIAL</option>
<option value="ROOM">ROOM</option>
</select>
</label>
<label>
offer_type
<select
value={valType}
onChange={(e) => setValType(e.target.value)}
>
<option value="SELL">SELL</option>
<option value="RENT">RENT</option>
</select>
</label>
<label>
page
<input
type="number"
min={1}
max={100}
value={valPage}
onChange={(e) => setValPage(e.target.value)}
required
/>
</label>
<button type="submit" disabled={valMut.isPending}>
{valMut.isPending ? "Запрашиваем…" : "Запустить"}
</button>
</form>
<ResultPanel mut={valMut} />
</section>
</main>
{/* 4. Valuation */}
<div className="scraper-section">
<h3>4. Valuation (anonymous history)</h3>
<p className="scraper-hint">
Anonymous GET /otsenka-kvartiry-po-adresu-onlayn/ house meta + до
30 history items на page.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
valMut.mutate({
address: valAddress,
offer_category: valCategory,
offer_type: valType,
page: parseInt(valPage, 10),
});
}}
>
<label className="full">
Адрес
<input
value={valAddress}
onChange={(e) => setValAddress(e.target.value)}
required
placeholder="Свердловская область, Екатеринбург, улица Учителей, 18"
/>
</label>
<label>
offer_category
<select
value={valCategory}
onChange={(e) => setValCategory(e.target.value)}
>
<option value="APARTMENT">APARTMENT</option>
<option value="HOUSE">HOUSE</option>
<option value="GARAGE">GARAGE</option>
<option value="COMMERCIAL">COMMERCIAL</option>
<option value="ROOM">ROOM</option>
</select>
</label>
<label>
offer_type
<select
value={valType}
onChange={(e) => setValType(e.target.value)}
>
<option value="SELL">SELL</option>
<option value="RENT">RENT</option>
</select>
</label>
<label>
page
<input
type="number"
min={1}
max={100}
value={valPage}
onChange={(e) => setValPage(e.target.value)}
required
/>
</label>
<button type="submit" disabled={valMut.isPending}>
{valMut.isPending ? "Запрашиваем…" : "Запустить"}
</button>
</form>
<ResultPanel mut={valMut} />
</div>
</>
);
}
// ── Page config + export ───────────────────────────────────────────────────
const YANDEX_CONFIG: ScraperPageConfig = {
source: "yandex",
activeTab: "yandex",
displayName: "Yandex скрапер",
subtitle:
"Ручной запуск sub-pipeline'ов YandexRealtyScraper v1. Для debug и теста после изменений.",
scheduleSource: "yandex_city_sweep",
paramConfig: {
hasDetailParams: false,
hasEnrichAddress: true,
defaults: {
pages_per_anchor: 3,
request_delay_sec: 5,
radius_m: 1500,
enrich_address: false,
},
},
delayMin: 1,
delayMax: 60,
};
export default function YandexScraperPage() {
return (
<ScraperPage
config={{
...YANDEX_CONFIG,
extraTriggers: <YandexExtraTriggers />,
}}
/>
);
}

View file

@ -9,10 +9,11 @@ import { safeUrl } from "@/lib/safeUrl";
import { useBrand } from "@/lib/useBrand";
import { useMe } from "@/lib/useMe";
type ActiveTab =
export type ActiveTab =
| "estimate"
| "history"
| "cache"
| "scrapers"
| "avito"
| "cian"
| "yandex"
@ -48,6 +49,12 @@ const NAV_ITEMS: Array<{
{ key: "history", href: "/history", scopePath: "/trade-in/history", label: "История" },
{ key: "cache", href: "/cache", scopePath: "/trade-in/cache", label: "Кэш" },
// Скраперы — admin-only UI. Маппим на admin-deny path, чтобы pilot их не видел.
{
key: "scrapers",
href: "/scrapers",
scopePath: "/trade-in/api/v1/admin/scrapers",
label: "Скрапперы",
},
{
key: "avito",
href: "/scrapers/avito",