feat(tradein): unify scraper admin pages (avito/cian/yandex) + /scrapers hub (#878)
All checks were successful
Deploy Trade-In / changes (push) Successful in 5s
Deploy Trade-In / test (push) Has been skipped
Deploy Trade-In / build-backend (push) Has been skipped
Deploy Trade-In / build-frontend (push) Successful in 1m37s
Deploy Trade-In / deploy (push) Successful in 36s
All checks were successful
Deploy Trade-In / changes (push) Successful in 5s
Deploy Trade-In / test (push) Has been skipped
Deploy Trade-In / build-backend (push) Has been skipped
Deploy Trade-In / build-frontend (push) Successful in 1m37s
Deploy Trade-In / deploy (push) Successful in 36s
Co-authored-by: bot-frontend <bot-frontend@gendsgn.local> Co-committed-by: bot-frontend <bot-frontend@gendsgn.local>
This commit is contained in:
parent
d9b8047a7d
commit
ee8f72da39
7 changed files with 2586 additions and 2079 deletions
|
|
@ -0,0 +1,967 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
|
||||
import "@/components/trade-in/trade-in.css";
|
||||
import { Topbar, type ActiveTab } from "@/components/trade-in/Topbar";
|
||||
import { apiFetch } from "@/lib/api";
|
||||
|
||||
// ── Shared Types ───────────────────────────────────────────────────────────
|
||||
|
||||
export interface ScrapeRunRow {
|
||||
run_id: number;
|
||||
source: string;
|
||||
status: string;
|
||||
params: Record<string, unknown> | null;
|
||||
counters: {
|
||||
anchors_total?: number;
|
||||
anchors_done?: number;
|
||||
lots_fetched?: number;
|
||||
lots_inserted?: number;
|
||||
lots_updated?: number;
|
||||
unique_houses?: number;
|
||||
houses_enriched?: number;
|
||||
houses_failed?: number;
|
||||
detail_attempted?: number;
|
||||
detail_enriched?: number;
|
||||
detail_failed?: number;
|
||||
errors_count?: number;
|
||||
} | null;
|
||||
error: string | null;
|
||||
started_at: string | null;
|
||||
finished_at: string | null;
|
||||
heartbeat_at: string | null;
|
||||
}
|
||||
|
||||
export interface ScheduleConfig {
|
||||
id: number;
|
||||
source: string;
|
||||
enabled: boolean;
|
||||
window_start_hour: number;
|
||||
window_end_hour: number;
|
||||
default_params: {
|
||||
pages_per_anchor?: number;
|
||||
detail_top_n?: number;
|
||||
request_delay_sec?: number;
|
||||
enrich_houses?: boolean;
|
||||
enrich_address?: boolean;
|
||||
radius_m?: number;
|
||||
};
|
||||
last_run_id: number | null;
|
||||
last_run_at: string | null;
|
||||
next_run_at: string | null;
|
||||
updated_at: string | null;
|
||||
}
|
||||
|
||||
export interface ScraperSetting {
|
||||
source: string;
|
||||
request_delay_sec: number;
|
||||
description: string | null;
|
||||
updated_at: string | null;
|
||||
}
|
||||
|
||||
export interface CitySweepStartResp {
|
||||
run_id: number;
|
||||
status: string;
|
||||
pages_per_anchor: number;
|
||||
detail_top_n: number;
|
||||
}
|
||||
|
||||
// ── Shared sweep param types ───────────────────────────────────────────────
|
||||
|
||||
export interface SweepParamConfig {
|
||||
/** has detail_top_n + enrich_houses params */
|
||||
hasDetailParams: boolean;
|
||||
/** has enrich_address param (yandex) */
|
||||
hasEnrichAddress: boolean;
|
||||
defaults: {
|
||||
pages_per_anchor: number;
|
||||
radius_m: number;
|
||||
request_delay_sec: number;
|
||||
detail_top_n?: number;
|
||||
enrich_houses?: boolean;
|
||||
enrich_address?: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
// ── Shared API source name type ────────────────────────────────────────────
|
||||
|
||||
export type ScraperSource = "avito" | "cian" | "yandex";
|
||||
export type { ActiveTab };
|
||||
export type ScheduleSourceKey =
|
||||
| "avito_city_sweep"
|
||||
| "cian_city_sweep"
|
||||
| "yandex_city_sweep";
|
||||
|
||||
// ── Shared Hooks ───────────────────────────────────────────────────────────
|
||||
|
||||
export function useStartCitySweep(source: ScraperSource) {
|
||||
const qc = useQueryClient();
|
||||
return useMutation<CitySweepStartResp, Error, Record<string, unknown>>({
|
||||
mutationFn: (input) =>
|
||||
apiFetch<CitySweepStartResp>(
|
||||
`/api/v1/admin/scrape/${source}-city-sweep`,
|
||||
{
|
||||
method: "POST",
|
||||
body: JSON.stringify(input),
|
||||
},
|
||||
),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ["city-sweep-runs", source] });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useCitySweepRuns(source: ScraperSource) {
|
||||
return useQuery<ScrapeRunRow[]>({
|
||||
queryKey: ["city-sweep-runs", source],
|
||||
queryFn: () =>
|
||||
apiFetch<ScrapeRunRow[]>(
|
||||
`/api/v1/admin/scrape/${source}-city-sweep/runs?limit=10`,
|
||||
),
|
||||
refetchInterval: 5_000,
|
||||
staleTime: 0,
|
||||
});
|
||||
}
|
||||
|
||||
export function useCancelCitySweep(source: ScraperSource) {
|
||||
const qc = useQueryClient();
|
||||
return useMutation<
|
||||
{ ok: boolean; run_id: number; cancelled: boolean },
|
||||
Error,
|
||||
number
|
||||
>({
|
||||
mutationFn: (run_id) =>
|
||||
apiFetch(
|
||||
`/api/v1/admin/scrape/${source}-city-sweep/${run_id}/cancel`,
|
||||
{ method: "POST" },
|
||||
),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ["city-sweep-runs", source] });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useSchedules() {
|
||||
return useQuery<ScheduleConfig[]>({
|
||||
queryKey: ["scrape-schedules"],
|
||||
queryFn: () =>
|
||||
apiFetch<ScheduleConfig[]>("/api/v1/admin/scrape/schedules"),
|
||||
refetchInterval: 30_000,
|
||||
});
|
||||
}
|
||||
|
||||
export function useUpdateSchedule() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation<
|
||||
ScheduleConfig,
|
||||
Error,
|
||||
{
|
||||
source: ScheduleSourceKey;
|
||||
payload: {
|
||||
enabled: boolean;
|
||||
window_start_hour: number;
|
||||
window_end_hour: number;
|
||||
default_params: Record<string, unknown>;
|
||||
};
|
||||
}
|
||||
>({
|
||||
mutationFn: ({ source, payload }) =>
|
||||
apiFetch<ScheduleConfig>(`/api/v1/admin/scrape/schedules/${source}`, {
|
||||
method: "PUT",
|
||||
body: JSON.stringify(payload),
|
||||
}),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ["scrape-schedules"] });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
export function useScraperSettings() {
|
||||
return useQuery<ScraperSetting[]>({
|
||||
queryKey: ["scraper-settings"],
|
||||
queryFn: () =>
|
||||
apiFetch<{ settings: ScraperSetting[] }>(
|
||||
"/api/v1/admin/scraper-settings",
|
||||
).then((r) =>
|
||||
(r.settings ?? []).map((s) => ({
|
||||
...s,
|
||||
request_delay_sec: Number(s.request_delay_sec),
|
||||
})),
|
||||
),
|
||||
staleTime: 30_000,
|
||||
refetchInterval: 60_000,
|
||||
});
|
||||
}
|
||||
|
||||
export function useUpdateScraperSetting() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation<
|
||||
ScraperSetting,
|
||||
Error,
|
||||
{ source: string; request_delay_sec: number }
|
||||
>({
|
||||
mutationFn: ({ source, request_delay_sec }) =>
|
||||
apiFetch<ScraperSetting>(`/api/v1/admin/scraper-settings/${source}`, {
|
||||
method: "PUT",
|
||||
body: JSON.stringify({ source, request_delay_sec }),
|
||||
}),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ["scraper-settings"] });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ── Shared Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
export function translateStatus(s: string): string {
|
||||
switch (s) {
|
||||
case "running":
|
||||
return "выполняется";
|
||||
case "done":
|
||||
return "готово";
|
||||
case "failed":
|
||||
return "ошибка";
|
||||
case "cancelled":
|
||||
return "отменено";
|
||||
case "zombie":
|
||||
return "зомби";
|
||||
case "skipped":
|
||||
return "пропущено";
|
||||
case "banned":
|
||||
return "блок";
|
||||
default:
|
||||
return s;
|
||||
}
|
||||
}
|
||||
|
||||
export function formatTime(iso: string | null): string {
|
||||
if (!iso) return "—";
|
||||
try {
|
||||
return new Date(iso).toLocaleString("ru-RU", {
|
||||
day: "2-digit",
|
||||
month: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
});
|
||||
} catch {
|
||||
return iso;
|
||||
}
|
||||
}
|
||||
|
||||
export function utcToMsk(hour: number): number {
|
||||
return (hour + 3) % 24;
|
||||
}
|
||||
|
||||
export function mskToUtc(hour: number): number {
|
||||
return (hour - 3 + 24) % 24;
|
||||
}
|
||||
|
||||
function formatMskRange(utcStart: number, utcEnd: number): string {
|
||||
const ms = utcToMsk(utcStart);
|
||||
const me = utcToMsk(utcEnd);
|
||||
return `${String(ms).padStart(2, "0")}:00–${String(me).padStart(2, "0")}:00 МСК`;
|
||||
}
|
||||
|
||||
// ── ResultPanel (reusable) ─────────────────────────────────────────────────
|
||||
|
||||
export interface ResultPanelProps<TData> {
|
||||
mut: {
|
||||
data?: TData;
|
||||
error: Error | null;
|
||||
isPending: boolean;
|
||||
isSuccess: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
export function ResultPanel<TData>({ mut }: ResultPanelProps<TData>) {
|
||||
if (mut.error) {
|
||||
return (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
<strong>Ошибка:</strong> {mut.error.message}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (mut.isSuccess && mut.data) {
|
||||
return (
|
||||
<div className="scraper-result">
|
||||
<pre>{JSON.stringify(mut.data, null, 2)}</pre>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Runs Log Section ───────────────────────────────────────────────────────
|
||||
|
||||
interface RunsLogSectionProps {
|
||||
source: ScraperSource;
|
||||
}
|
||||
|
||||
function RunsLogSection({ source }: RunsLogSectionProps) {
|
||||
const runsQ = useCitySweepRuns(source);
|
||||
const cancelMut = useCancelCitySweep(source);
|
||||
|
||||
return (
|
||||
<section className="scraper-section scraper-section--highlight">
|
||||
<h2>Очередь / Лог прогонов</h2>
|
||||
<p className="scraper-hint">
|
||||
Автообновление каждые 5 секунд. Показаны последние 10 прогонов.
|
||||
</p>
|
||||
<div className="city-sweep-runs">
|
||||
<h3>Recent runs (auto-refresh 5s)</h3>
|
||||
{runsQ.isPending && <p className="scraper-hint">Загрузка…</p>}
|
||||
{runsQ.error && (
|
||||
<p className="scraper-result scraper-result--error">
|
||||
Ошибка загрузки runs: {runsQ.error.message}
|
||||
</p>
|
||||
)}
|
||||
{runsQ.data && runsQ.data.length === 0 && (
|
||||
<p className="scraper-hint">Нет запусков пока.</p>
|
||||
)}
|
||||
{runsQ.data && runsQ.data.length > 0 && (
|
||||
<table className="runs-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th>Статус</th>
|
||||
<th>Старт</th>
|
||||
<th>Heartbeat</th>
|
||||
<th>Anchors</th>
|
||||
<th>Lots</th>
|
||||
<th>Houses</th>
|
||||
<th>Detail</th>
|
||||
<th>Errors</th>
|
||||
<th>Действие</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{runsQ.data.map((r) => (
|
||||
<tr key={r.run_id} className={`run-row run-row--${r.status}`}>
|
||||
<td>{r.run_id}</td>
|
||||
<td>
|
||||
<span className={`run-badge run-badge--${r.status}`}>
|
||||
{translateStatus(r.status)}
|
||||
</span>
|
||||
</td>
|
||||
<td>{formatTime(r.started_at)}</td>
|
||||
<td>{formatTime(r.heartbeat_at)}</td>
|
||||
<td>
|
||||
{r.counters?.anchors_done ?? 0}/
|
||||
{r.counters?.anchors_total ?? 0}
|
||||
</td>
|
||||
<td>
|
||||
{r.counters?.lots_fetched ?? 0}{" "}
|
||||
<span className="run-muted">
|
||||
(+{r.counters?.lots_inserted ?? 0})
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
{r.counters?.houses_enriched ?? 0}/
|
||||
{r.counters?.unique_houses ?? 0}
|
||||
</td>
|
||||
<td>
|
||||
{r.counters?.detail_enriched ?? 0}/
|
||||
{r.counters?.detail_attempted ?? 0}
|
||||
</td>
|
||||
<td>{r.counters?.errors_count ?? 0}</td>
|
||||
<td>
|
||||
{r.status === "running" && (
|
||||
<button
|
||||
type="button"
|
||||
className="cancel-btn"
|
||||
disabled={
|
||||
cancelMut.isPending &&
|
||||
cancelMut.variables === r.run_id
|
||||
}
|
||||
onClick={() => cancelMut.mutate(r.run_id)}
|
||||
>
|
||||
Отменить
|
||||
</button>
|
||||
)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Cron Schedule Section ──────────────────────────────────────────────────
|
||||
|
||||
interface CronScheduleSectionProps {
|
||||
scheduleSource: ScheduleSourceKey;
|
||||
paramConfig: SweepParamConfig;
|
||||
}
|
||||
|
||||
function CronScheduleSection({
|
||||
scheduleSource,
|
||||
paramConfig,
|
||||
}: CronScheduleSectionProps) {
|
||||
const schedulesQ = useSchedules();
|
||||
const updateScheduleMut = useUpdateSchedule();
|
||||
|
||||
const [schEnabled, setSchEnabled] = useState(true);
|
||||
const [schMskStart, setSchMskStart] = useState(5);
|
||||
const [schMskEnd, setSchMskEnd] = useState(8);
|
||||
const [schPages, setSchPages] = useState(
|
||||
paramConfig.defaults.pages_per_anchor,
|
||||
);
|
||||
const [schTopN, setSchTopN] = useState(paramConfig.defaults.detail_top_n ?? 0);
|
||||
const [schDelay, setSchDelay] = useState(
|
||||
paramConfig.defaults.request_delay_sec,
|
||||
);
|
||||
const [schRadius, setSchRadius] = useState(paramConfig.defaults.radius_m);
|
||||
const [schEnrichHouses, setSchEnrichHouses] = useState(
|
||||
paramConfig.defaults.enrich_houses ?? false,
|
||||
);
|
||||
const [schEnrichAddress, setSchEnrichAddress] = useState(
|
||||
paramConfig.defaults.enrich_address ?? false,
|
||||
);
|
||||
|
||||
useEffect(() => {
|
||||
const sweep = schedulesQ.data?.find((s) => s.source === scheduleSource);
|
||||
if (!sweep) return;
|
||||
setSchEnabled(sweep.enabled);
|
||||
setSchMskStart(utcToMsk(sweep.window_start_hour));
|
||||
setSchMskEnd(utcToMsk(sweep.window_end_hour));
|
||||
const p = sweep.default_params ?? {};
|
||||
if (typeof p.pages_per_anchor === "number")
|
||||
setSchPages(p.pages_per_anchor);
|
||||
if (typeof p.detail_top_n === "number") setSchTopN(p.detail_top_n);
|
||||
if (typeof p.request_delay_sec === "number")
|
||||
setSchDelay(p.request_delay_sec);
|
||||
if (typeof p.radius_m === "number") setSchRadius(p.radius_m);
|
||||
if (typeof p.enrich_houses === "boolean") setSchEnrichHouses(p.enrich_houses);
|
||||
if (typeof p.enrich_address === "boolean")
|
||||
setSchEnrichAddress(p.enrich_address);
|
||||
}, [schedulesQ.data, scheduleSource]);
|
||||
|
||||
const currentSweep = schedulesQ.data?.find(
|
||||
(s) => s.source === scheduleSource,
|
||||
);
|
||||
|
||||
return (
|
||||
<section className="scraper-section scraper-section--schedule">
|
||||
<h2>Cron-расписание автозапуска (in-app scheduler)</h2>
|
||||
<p className="scraper-hint">
|
||||
Backend периодически (каждую минуту) проверяет это расписание и
|
||||
запускает city sweep автоматически в случайное время в указанном окне.
|
||||
Без SSH, без crontab — всё в БД и UI.
|
||||
</p>
|
||||
|
||||
{schedulesQ.isPending && <p className="scraper-hint">Загрузка…</p>}
|
||||
{schedulesQ.error && (
|
||||
<p className="scraper-result scraper-result--error">
|
||||
Ошибка загрузки: {schedulesQ.error.message}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{currentSweep && (
|
||||
<div className="schedule-status">
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Статус:</span>
|
||||
<span
|
||||
className={`run-badge run-badge--${currentSweep.enabled ? "running" : "cancelled"}`}
|
||||
>
|
||||
{currentSweep.enabled ? "включено" : "отключено"}
|
||||
</span>
|
||||
</div>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Окно (МСК):</span>
|
||||
<span>
|
||||
{formatMskRange(
|
||||
currentSweep.window_start_hour,
|
||||
currentSweep.window_end_hour,
|
||||
)}
|
||||
</span>
|
||||
<span className="run-muted">
|
||||
({String(currentSweep.window_start_hour).padStart(2, "0")}:00–
|
||||
{String(currentSweep.window_end_hour).padStart(2, "0")}:00 UTC)
|
||||
</span>
|
||||
</div>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Следующий запуск:</span>
|
||||
<span>{formatTime(currentSweep.next_run_at)}</span>
|
||||
</div>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Последний запуск:</span>
|
||||
<span>
|
||||
{currentSweep.last_run_at
|
||||
? formatTime(currentSweep.last_run_at)
|
||||
: "—"}
|
||||
{currentSweep.last_run_id &&
|
||||
` (run #${currentSweep.last_run_id})`}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
const defaultParams: Record<string, unknown> = {
|
||||
pages_per_anchor: schPages,
|
||||
request_delay_sec: schDelay,
|
||||
radius_m: schRadius,
|
||||
};
|
||||
if (paramConfig.hasDetailParams) {
|
||||
defaultParams.detail_top_n = schTopN;
|
||||
defaultParams.enrich_houses = schEnrichHouses;
|
||||
}
|
||||
if (paramConfig.hasEnrichAddress) {
|
||||
defaultParams.enrich_address = schEnrichAddress;
|
||||
}
|
||||
updateScheduleMut.mutate({
|
||||
source: scheduleSource,
|
||||
payload: {
|
||||
enabled: schEnabled,
|
||||
window_start_hour: mskToUtc(schMskStart),
|
||||
window_end_hour: mskToUtc(schMskEnd),
|
||||
default_params: defaultParams,
|
||||
},
|
||||
});
|
||||
}}
|
||||
>
|
||||
<label className="checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={schEnabled}
|
||||
onChange={(e) => setSchEnabled(e.target.checked)}
|
||||
/>
|
||||
Включено
|
||||
</label>
|
||||
<label>
|
||||
Окно начала (МСК)
|
||||
<input
|
||||
type="number"
|
||||
min={0}
|
||||
max={23}
|
||||
value={schMskStart}
|
||||
onChange={(e) => setSchMskStart(parseInt(e.target.value, 10))}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Окно окончания (МСК)
|
||||
<input
|
||||
type="number"
|
||||
min={0}
|
||||
max={23}
|
||||
value={schMskEnd}
|
||||
onChange={(e) => setSchMskEnd(parseInt(e.target.value, 10))}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Pages/anchor
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={10}
|
||||
value={schPages}
|
||||
onChange={(e) => setSchPages(parseInt(e.target.value, 10))}
|
||||
/>
|
||||
</label>
|
||||
{paramConfig.hasDetailParams && (
|
||||
<>
|
||||
<label>
|
||||
Detail top-N
|
||||
<input
|
||||
type="number"
|
||||
min={0}
|
||||
max={30}
|
||||
value={schTopN}
|
||||
onChange={(e) => setSchTopN(parseInt(e.target.value, 10))}
|
||||
/>
|
||||
</label>
|
||||
</>
|
||||
)}
|
||||
<label>
|
||||
Delay (сек)
|
||||
<input
|
||||
type="number"
|
||||
step={0.5}
|
||||
min={3}
|
||||
max={15}
|
||||
value={schDelay}
|
||||
onChange={(e) => setSchDelay(parseFloat(e.target.value))}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Радиус (м)
|
||||
<input
|
||||
type="number"
|
||||
min={500}
|
||||
max={5000}
|
||||
step={100}
|
||||
value={schRadius}
|
||||
onChange={(e) => setSchRadius(parseInt(e.target.value, 10))}
|
||||
/>
|
||||
</label>
|
||||
{paramConfig.hasDetailParams && (
|
||||
<label className="checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={schEnrichHouses}
|
||||
onChange={(e) => setSchEnrichHouses(e.target.checked)}
|
||||
/>
|
||||
Enrich houses
|
||||
</label>
|
||||
)}
|
||||
{paramConfig.hasEnrichAddress && (
|
||||
<label className="checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={schEnrichAddress}
|
||||
onChange={(e) => setSchEnrichAddress(e.target.checked)}
|
||||
/>
|
||||
Enrich address
|
||||
</label>
|
||||
)}
|
||||
<button type="submit" disabled={updateScheduleMut.isPending}>
|
||||
{updateScheduleMut.isPending
|
||||
? "Сохраняем…"
|
||||
: "Сохранить расписание"}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
{updateScheduleMut.error && (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
Ошибка: {updateScheduleMut.error.message}
|
||||
</div>
|
||||
)}
|
||||
{updateScheduleMut.isSuccess && updateScheduleMut.data && (
|
||||
<div className="scraper-result">
|
||||
Расписание обновлено. Next run:{" "}
|
||||
{formatTime(updateScheduleMut.data.next_run_at)}
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Per-source Delay Section ───────────────────────────────────────────────
|
||||
|
||||
interface PerSourceDelaySectionProps {
|
||||
source: ScraperSource;
|
||||
delayMin?: number;
|
||||
delayMax?: number;
|
||||
}
|
||||
|
||||
function PerSourceDelaySection({
|
||||
source,
|
||||
delayMin = 1,
|
||||
delayMax = 60,
|
||||
}: PerSourceDelaySectionProps) {
|
||||
const settingsQ = useScraperSettings();
|
||||
const updateMut = useUpdateScraperSetting();
|
||||
|
||||
const currentSetting = settingsQ.data?.find((s) => s.source === source);
|
||||
const [delayInput, setDelayInput] = useState<string>("5.0");
|
||||
const [initialized, setInitialized] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (currentSetting && !initialized) {
|
||||
setDelayInput(String(currentSetting.request_delay_sec));
|
||||
setInitialized(true);
|
||||
}
|
||||
}, [currentSetting, initialized]);
|
||||
|
||||
return (
|
||||
<section className="scraper-section">
|
||||
<h2>Задержка запросов (этот скраппер)</h2>
|
||||
<p className="scraper-hint">
|
||||
Задержка применяется только к скрапперу <strong>{source}</strong>.
|
||||
Изменение вступает в силу немедленно без рестарта backend. Глобальный
|
||||
нижний порог настраивается на{" "}
|
||||
<a href="/scrapers">странице хаба скрапперов</a>.
|
||||
</p>
|
||||
|
||||
{settingsQ.isPending && (
|
||||
<p className="scraper-hint">Загрузка настроек…</p>
|
||||
)}
|
||||
{settingsQ.error && (
|
||||
<p className="scraper-result scraper-result--error">
|
||||
Ошибка загрузки: {settingsQ.error.message}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{currentSetting && (
|
||||
<div className="schedule-status" style={{ marginBottom: "12px" }}>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Текущая задержка:</span>
|
||||
<span>
|
||||
<strong>{currentSetting.request_delay_sec} сек</strong>
|
||||
</span>
|
||||
</div>
|
||||
{currentSetting.updated_at && (
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Обновлено:</span>
|
||||
<span>{formatTime(currentSetting.updated_at)}</span>
|
||||
</div>
|
||||
)}
|
||||
{currentSetting.description && (
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Описание:</span>
|
||||
<span className="run-muted">{currentSetting.description}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
updateMut.mutate({
|
||||
source,
|
||||
request_delay_sec: parseFloat(delayInput),
|
||||
});
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
Задержка (сек, {delayMin}–{delayMax})
|
||||
<input
|
||||
type="number"
|
||||
min={delayMin}
|
||||
max={delayMax}
|
||||
step={0.5}
|
||||
value={delayInput}
|
||||
onChange={(e) => setDelayInput(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={updateMut.isPending}>
|
||||
{updateMut.isPending ? "Сохраняем…" : "Сохранить задержку"}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
{updateMut.error && (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
<strong>Ошибка:</strong> {updateMut.error.message}
|
||||
</div>
|
||||
)}
|
||||
{updateMut.isSuccess && updateMut.data && (
|
||||
<div className="scraper-result">
|
||||
Сохранено. Задержка:{" "}
|
||||
<strong>
|
||||
{Number(updateMut.data.request_delay_sec).toFixed(1)} сек
|
||||
</strong>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// ── City Sweep Section (shared trigger) ───────────────────────────────────
|
||||
|
||||
interface CitySweepSectionProps {
|
||||
source: ScraperSource;
|
||||
paramConfig: SweepParamConfig;
|
||||
}
|
||||
|
||||
function CitySweepSection({ source, paramConfig }: CitySweepSectionProps) {
|
||||
const sweepMut = useStartCitySweep(source);
|
||||
|
||||
const [sweepPages, setSweepPages] = useState(
|
||||
String(paramConfig.defaults.pages_per_anchor),
|
||||
);
|
||||
const [sweepTopN, setSweepTopN] = useState(
|
||||
String(paramConfig.defaults.detail_top_n ?? 0),
|
||||
);
|
||||
const [sweepDelay, setSweepDelay] = useState(
|
||||
String(paramConfig.defaults.request_delay_sec),
|
||||
);
|
||||
const [sweepRadius, setSweepRadius] = useState(
|
||||
String(paramConfig.defaults.radius_m),
|
||||
);
|
||||
const [sweepEnrichHouses, setSweepEnrichHouses] = useState(
|
||||
paramConfig.defaults.enrich_houses ?? false,
|
||||
);
|
||||
const [sweepEnrichAddress, setSweepEnrichAddress] = useState(
|
||||
paramConfig.defaults.enrich_address ?? false,
|
||||
);
|
||||
|
||||
return (
|
||||
<section className="scraper-section">
|
||||
<h2>Полный обход ЕКБ (city sweep)</h2>
|
||||
<p className="scraper-hint">
|
||||
Iterate anchors × N pages → save listings → enrich. Запускается в
|
||||
background. Можно отменить в логе прогонов ниже.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
const body: Record<string, unknown> = {
|
||||
pages_per_anchor: parseInt(sweepPages, 10),
|
||||
request_delay_sec: parseFloat(sweepDelay),
|
||||
radius_m: parseInt(sweepRadius, 10),
|
||||
};
|
||||
if (paramConfig.hasDetailParams) {
|
||||
body.detail_top_n = parseInt(sweepTopN, 10);
|
||||
body.enrich_houses = sweepEnrichHouses;
|
||||
}
|
||||
if (paramConfig.hasEnrichAddress) {
|
||||
body.enrich_address = sweepEnrichAddress;
|
||||
}
|
||||
sweepMut.mutate(body);
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
Страниц на anchor
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={10}
|
||||
value={sweepPages}
|
||||
onChange={(e) => setSweepPages(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
{paramConfig.hasDetailParams && (
|
||||
<label>
|
||||
Detail top-N
|
||||
<input
|
||||
type="number"
|
||||
min={0}
|
||||
max={30}
|
||||
value={sweepTopN}
|
||||
onChange={(e) => setSweepTopN(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
)}
|
||||
<label>
|
||||
Delay (сек)
|
||||
<input
|
||||
type="number"
|
||||
step={0.5}
|
||||
min={3}
|
||||
max={15}
|
||||
value={sweepDelay}
|
||||
onChange={(e) => setSweepDelay(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Радиус (м)
|
||||
<input
|
||||
type="number"
|
||||
min={500}
|
||||
max={5000}
|
||||
step={100}
|
||||
value={sweepRadius}
|
||||
onChange={(e) => setSweepRadius(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
{paramConfig.hasDetailParams && (
|
||||
<label className="checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={sweepEnrichHouses}
|
||||
onChange={(e) => setSweepEnrichHouses(e.target.checked)}
|
||||
/>
|
||||
Enrich houses
|
||||
</label>
|
||||
)}
|
||||
{paramConfig.hasEnrichAddress && (
|
||||
<label className="checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={sweepEnrichAddress}
|
||||
onChange={(e) => setSweepEnrichAddress(e.target.checked)}
|
||||
/>
|
||||
Enrich address
|
||||
</label>
|
||||
)}
|
||||
<button type="submit" disabled={sweepMut.isPending}>
|
||||
{sweepMut.isPending ? "Запускаем…" : "Запустить sweep"}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
{sweepMut.error && (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
<strong>Ошибка:</strong> {sweepMut.error.message}
|
||||
</div>
|
||||
)}
|
||||
{sweepMut.isSuccess && sweepMut.data && (
|
||||
<div className="scraper-result">
|
||||
<pre>{JSON.stringify(sweepMut.data, null, 2)}</pre>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// ── ScraperPage config ─────────────────────────────────────────────────────
|
||||
|
||||
export interface ScraperPageConfig {
|
||||
source: ScraperSource;
|
||||
activeTab: ActiveTab;
|
||||
displayName: string;
|
||||
subtitle: string;
|
||||
scheduleSource: ScheduleSourceKey;
|
||||
paramConfig: SweepParamConfig;
|
||||
delayMin?: number;
|
||||
delayMax?: number;
|
||||
/** Extra trigger forms rendered inside the "Запуск" section */
|
||||
extraTriggers?: React.ReactNode;
|
||||
/** Extra tabs/sections rendered after the standard 4 sections */
|
||||
extraSections?: React.ReactNode;
|
||||
}
|
||||
|
||||
// ── ScraperPage (main export) ──────────────────────────────────────────────
|
||||
|
||||
export function ScraperPage({ config }: { config: ScraperPageConfig }) {
|
||||
const {
|
||||
source,
|
||||
activeTab,
|
||||
displayName,
|
||||
subtitle,
|
||||
scheduleSource,
|
||||
paramConfig,
|
||||
delayMin,
|
||||
delayMax,
|
||||
extraTriggers,
|
||||
extraSections,
|
||||
} = config;
|
||||
|
||||
return (
|
||||
<>
|
||||
<Topbar active={activeTab} />
|
||||
<main className="page scraper-page">
|
||||
<h1 className="scraper-h1">{displayName} — admin триггер</h1>
|
||||
<p className="scraper-subtitle">{subtitle}</p>
|
||||
|
||||
{/* Section 1: Запуск — city sweep + extra provider triggers */}
|
||||
<section className="scraper-section">
|
||||
<h2>Запуск</h2>
|
||||
{extraTriggers}
|
||||
<CitySweepSection source={source} paramConfig={paramConfig} />
|
||||
</section>
|
||||
|
||||
{/* Section 2: Runs log */}
|
||||
<RunsLogSection source={source} />
|
||||
|
||||
{/* Section 3: Cron schedule */}
|
||||
<CronScheduleSection
|
||||
scheduleSource={scheduleSource}
|
||||
paramConfig={paramConfig}
|
||||
/>
|
||||
|
||||
{/* Section 4: Per-source delay */}
|
||||
<PerSourceDelaySection
|
||||
source={source}
|
||||
delayMin={delayMin}
|
||||
delayMax={delayMax}
|
||||
/>
|
||||
|
||||
{/* Extra sections (e.g. cookies tab for cian) */}
|
||||
{extraSections}
|
||||
</main>
|
||||
</>
|
||||
);
|
||||
}
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -1,351 +1,21 @@
|
|||
"use client";
|
||||
|
||||
import { useState } from "react";
|
||||
import { useMutation, useQuery } from "@tanstack/react-query";
|
||||
import { useEffect } from "react";
|
||||
import { useRouter } from "next/navigation";
|
||||
|
||||
import "@/components/trade-in/trade-in.css";
|
||||
import { Topbar } from "@/components/trade-in/Topbar";
|
||||
import { apiFetch } from "@/lib/api";
|
||||
|
||||
interface UploadResponse {
|
||||
ok?: boolean;
|
||||
userId?: number;
|
||||
cookieCount?: number;
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
interface TestAuthResponse {
|
||||
authenticated: boolean;
|
||||
userId?: number;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
interface CookieEditorEntry {
|
||||
name: string;
|
||||
value: string | number | boolean;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
function convertCookiesFormat(input: string): Record<string, string> {
|
||||
const parsed: unknown = JSON.parse(input.trim());
|
||||
if (Array.isArray(parsed)) {
|
||||
// Cookie-Editor array format: [{name, value, domain, ...}, ...]
|
||||
return (parsed as CookieEditorEntry[]).reduce<Record<string, string>>(
|
||||
(acc, c) => {
|
||||
if (
|
||||
typeof c === "object" &&
|
||||
c !== null &&
|
||||
"name" in c &&
|
||||
"value" in c
|
||||
) {
|
||||
acc[String(c.name)] = String(c.value);
|
||||
}
|
||||
return acc;
|
||||
},
|
||||
{},
|
||||
);
|
||||
}
|
||||
if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) {
|
||||
// Already a dict: {"name1": "val1", "name2": "val2"}
|
||||
return Object.fromEntries(
|
||||
Object.entries(parsed as Record<string, unknown>).map(([k, v]) => [
|
||||
k,
|
||||
String(v),
|
||||
]),
|
||||
);
|
||||
}
|
||||
throw new Error("Expected JSON array or plain object");
|
||||
}
|
||||
|
||||
function parsePreview(raw: string): {
|
||||
cookies: Record<string, string> | null;
|
||||
error: string | null;
|
||||
} {
|
||||
if (!raw.trim()) return { cookies: null, error: null };
|
||||
try {
|
||||
const cookies = convertCookiesFormat(raw);
|
||||
return { cookies, error: null };
|
||||
} catch (e) {
|
||||
return { cookies: null, error: e instanceof Error ? e.message : String(e) };
|
||||
}
|
||||
}
|
||||
|
||||
export default function CianCookiesPage() {
|
||||
const [rawInput, setRawInput] = useState("");
|
||||
const [showInstructions, setShowInstructions] = useState(false);
|
||||
|
||||
const preview = parsePreview(rawInput);
|
||||
const previewKeys = preview.cookies ? Object.keys(preview.cookies) : [];
|
||||
|
||||
const uploadMutation = useMutation({
|
||||
mutationFn: () => {
|
||||
if (!preview.cookies) {
|
||||
throw new Error("Invalid JSON — fix the input first");
|
||||
}
|
||||
return apiFetch<UploadResponse>(
|
||||
"/api/v1/admin/scrape/cian/upload-cookies",
|
||||
{
|
||||
method: "POST",
|
||||
body: JSON.stringify(preview.cookies),
|
||||
},
|
||||
);
|
||||
},
|
||||
onSuccess: () => setRawInput(""),
|
||||
});
|
||||
|
||||
const testAuthQuery = useQuery({
|
||||
queryKey: ["cian-test-auth"],
|
||||
queryFn: () =>
|
||||
apiFetch<TestAuthResponse>("/api/v1/admin/scrape/cian/test-auth"),
|
||||
enabled: false, // only on explicit button click
|
||||
retry: false,
|
||||
});
|
||||
|
||||
const canUpload =
|
||||
!!preview.cookies && previewKeys.length > 0 && !uploadMutation.isPending;
|
||||
/**
|
||||
* /scrapers/cian-cookies redirects to /scrapers/cian where cookie management
|
||||
* is now integrated as a dedicated section at the bottom of the Cian page.
|
||||
*/
|
||||
export default function CianCookiesRedirect() {
|
||||
const router = useRouter();
|
||||
useEffect(() => {
|
||||
router.replace("/scrapers/cian");
|
||||
}, [router]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Topbar active="cian-cookies" />
|
||||
<main className="page scraper-page">
|
||||
<h1 className="scraper-h1">Cian — обновление cookies</h1>
|
||||
<p className="scraper-subtitle">
|
||||
Позволяет обновить сессию Cian для скрапера оценщика без перезапуска
|
||||
backend. Endpoints:{" "}
|
||||
<code>POST /api/v1/admin/scrape/cian/upload-cookies</code> ·{" "}
|
||||
<code>GET /api/v1/admin/scrape/cian/test-auth</code>
|
||||
</p>
|
||||
|
||||
{/* Instructions collapsible */}
|
||||
<div className="scraper-section">
|
||||
<button
|
||||
onClick={() => setShowInstructions((v) => !v)}
|
||||
style={{
|
||||
background: "none",
|
||||
border: "none",
|
||||
padding: 0,
|
||||
cursor: "pointer",
|
||||
fontSize: 14,
|
||||
fontWeight: 600,
|
||||
color: "var(--accent, #1d4ed8)",
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
}}
|
||||
>
|
||||
<span>{showInstructions ? "▼" : "▶"}</span>
|
||||
Как получить cookies (инструкция)
|
||||
</button>
|
||||
{showInstructions && (
|
||||
<ol
|
||||
style={{
|
||||
marginTop: 12,
|
||||
paddingLeft: 20,
|
||||
fontSize: 13,
|
||||
color: "var(--fg, #374151)",
|
||||
lineHeight: 1.7,
|
||||
}}
|
||||
>
|
||||
<li>
|
||||
Открой{" "}
|
||||
<a
|
||||
href="https://www.cian.ru/"
|
||||
target="_blank"
|
||||
rel="noopener noreferrer"
|
||||
>
|
||||
cian.ru
|
||||
</a>{" "}
|
||||
и войди под нужным аккаунтом.
|
||||
</li>
|
||||
<li>
|
||||
Установи расширение <strong>Cookie-Editor</strong>{" "}
|
||||
(Chrome/Firefox) <em>или</em> открой DevTools → Application →
|
||||
Cookies → www.cian.ru.
|
||||
</li>
|
||||
<li>
|
||||
<strong>Cookie-Editor:</strong> нажми Export → Export as JSON →
|
||||
скопируй массив{" "}
|
||||
<code>[{"{"}name, value, ...{"}"}]</code>.
|
||||
</li>
|
||||
<li>
|
||||
<strong>DevTools (ручной способ):</strong> собери объект{" "}
|
||||
<code>
|
||||
{"{"}
|
||||
{'"'}name1{'"'}: {'"'}val1{'"'}, ...{"}"}
|
||||
</code>
|
||||
.
|
||||
</li>
|
||||
<li>
|
||||
Вставь JSON в поле ниже — формат определится автоматически.
|
||||
</li>
|
||||
<li>
|
||||
Убедись что preview показывает нужные ключи, затем нажми
|
||||
Upload.
|
||||
</li>
|
||||
</ol>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Cookie input */}
|
||||
<div className="scraper-section">
|
||||
<h2>Cookie JSON</h2>
|
||||
<p className="scraper-hint">
|
||||
Вставьте Cookie-Editor array или plain dict. Формат определяется
|
||||
автоматически.
|
||||
</p>
|
||||
<div className="scraper-form">
|
||||
<label className="full">
|
||||
<textarea
|
||||
value={rawInput}
|
||||
onChange={(e) => {
|
||||
setRawInput(e.target.value);
|
||||
uploadMutation.reset();
|
||||
}}
|
||||
rows={15}
|
||||
placeholder={`Формат 1 — Cookie-Editor array:\n[\n {"name": "session_key", "value": "abc123", "domain": ".cian.ru"},\n {"name": "_CSRF", "value": "xyz", "domain": ".cian.ru"}\n]\n\nФормат 2 — dict (DevTools manual):\n{\n "session_key": "abc123",\n "_CSRF": "xyz"\n}`}
|
||||
style={{
|
||||
fontFamily: "var(--font-mono, ui-monospace, monospace)",
|
||||
fontSize: 12,
|
||||
resize: "vertical",
|
||||
minHeight: 240,
|
||||
padding: "8px 10px",
|
||||
border: "1px solid var(--border-strong, #d1d5db)",
|
||||
borderRadius: 6,
|
||||
background: "#fff",
|
||||
color: "var(--fg, #111)",
|
||||
width: "100%",
|
||||
}}
|
||||
/>
|
||||
</label>
|
||||
|
||||
{/* Preview */}
|
||||
{rawInput.trim() && (
|
||||
<div
|
||||
className={`scraper-result${preview.error ? " scraper-result--error" : ""}`}
|
||||
style={{ fontFamily: "inherit" }}
|
||||
>
|
||||
{preview.error ? (
|
||||
<span>JSON error: {preview.error}</span>
|
||||
) : (
|
||||
<span style={{ color: "var(--success, #0a7a3a)" }}>
|
||||
Detected <strong>{previewKeys.length} cookies</strong>:{" "}
|
||||
{previewKeys.slice(0, 8).join(", ")}
|
||||
{previewKeys.length > 8
|
||||
? ` … +${previewKeys.length - 8} more`
|
||||
: ""}
|
||||
</span>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<button
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
uploadMutation.mutate();
|
||||
}}
|
||||
disabled={!canUpload}
|
||||
title={!preview.cookies ? "Вставьте корректный JSON" : undefined}
|
||||
>
|
||||
{uploadMutation.isPending ? "Uploading…" : "Upload Cookies"}
|
||||
</button>
|
||||
|
||||
<button
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
void testAuthQuery.refetch();
|
||||
}}
|
||||
disabled={testAuthQuery.isFetching}
|
||||
style={{
|
||||
background: "var(--surface-2, #f3f4f6)",
|
||||
color: "var(--fg, #1f2937)",
|
||||
border: "1px solid var(--border-strong, #d1d5db)",
|
||||
justifySelf: "start",
|
||||
}}
|
||||
>
|
||||
{testAuthQuery.isFetching ? "Checking…" : "Test Current Session"}
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Upload result */}
|
||||
{uploadMutation.isSuccess && (
|
||||
<div
|
||||
className="scraper-result"
|
||||
style={{
|
||||
fontFamily: "inherit",
|
||||
background: uploadMutation.data.ok
|
||||
? "var(--success-soft, #f0fdf4)"
|
||||
: "var(--danger-soft, #fef2f2)",
|
||||
borderColor: uploadMutation.data.ok
|
||||
? "var(--success, #0a7a3a)"
|
||||
: "var(--danger, #b3261e)",
|
||||
color: uploadMutation.data.ok
|
||||
? "var(--success, #0a7a3a)"
|
||||
: "var(--danger, #b3261e)",
|
||||
}}
|
||||
>
|
||||
{uploadMutation.data.ok ? (
|
||||
<>
|
||||
<strong>Cookies uploaded.</strong> userId={" "}
|
||||
{uploadMutation.data.userId ?? "—"} · cookieCount={" "}
|
||||
{uploadMutation.data.cookieCount ?? previewKeys.length}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<strong>Upload failed:</strong>{" "}
|
||||
{uploadMutation.data.detail ?? "Unknown error"}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{uploadMutation.isError && (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
<strong>Error:</strong> {String(uploadMutation.error)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Test auth result */}
|
||||
{(testAuthQuery.isSuccess || testAuthQuery.isError) && (
|
||||
<div
|
||||
className="scraper-result"
|
||||
style={{
|
||||
fontFamily: "inherit",
|
||||
background:
|
||||
testAuthQuery.isSuccess && testAuthQuery.data.authenticated
|
||||
? "var(--success-soft, #f0fdf4)"
|
||||
: "var(--danger-soft, #fef2f2)",
|
||||
borderColor:
|
||||
testAuthQuery.isSuccess && testAuthQuery.data.authenticated
|
||||
? "var(--success, #0a7a3a)"
|
||||
: "var(--danger, #b3261e)",
|
||||
color:
|
||||
testAuthQuery.isSuccess && testAuthQuery.data.authenticated
|
||||
? "var(--success, #0a7a3a)"
|
||||
: "var(--danger, #b3261e)",
|
||||
}}
|
||||
>
|
||||
{testAuthQuery.isError ? (
|
||||
<>
|
||||
<strong>Test auth error:</strong>{" "}
|
||||
{String(testAuthQuery.error)}
|
||||
</>
|
||||
) : testAuthQuery.data.authenticated ? (
|
||||
<>
|
||||
<strong>Authenticated.</strong> userId={" "}
|
||||
{testAuthQuery.data.userId ?? "—"}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<strong>Not authenticated.</strong>{" "}
|
||||
{testAuthQuery.data.reason
|
||||
? `Reason: ${testAuthQuery.data.reason}`
|
||||
: "Session expired or cookies invalid."}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</main>
|
||||
</>
|
||||
<p style={{ padding: "2rem", fontFamily: "sans-serif" }}>
|
||||
Перенаправление на страницу Cian скраппера…
|
||||
</p>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
423
tradein-mvp/frontend/src/app/scrapers/page.tsx
Normal file
423
tradein-mvp/frontend/src/app/scrapers/page.tsx
Normal file
|
|
@ -0,0 +1,423 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
|
||||
import "@/components/trade-in/trade-in.css";
|
||||
import { Topbar } from "@/components/trade-in/Topbar";
|
||||
import { API_BASE_URL } from "@/lib/api";
|
||||
import {
|
||||
formatTime,
|
||||
translateStatus,
|
||||
useScraperSettings,
|
||||
useSchedules,
|
||||
useUpdateScraperSetting,
|
||||
useUpdateSchedule,
|
||||
utcToMsk,
|
||||
type ScheduleConfig,
|
||||
type ScheduleSourceKey,
|
||||
} from "@/app/scrapers/_components/ScraperPage";
|
||||
|
||||
// ── Global delay section ───────────────────────────────────────────────────
|
||||
|
||||
function GlobalDelaySetting() {
|
||||
const settingsQ = useScraperSettings();
|
||||
const updateMut = useUpdateScraperSetting();
|
||||
|
||||
const globalSetting = settingsQ.data?.find((s) => s.source === "global");
|
||||
const [delayInput, setDelayInput] = useState("0");
|
||||
const [initialized, setInitialized] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (globalSetting && !initialized) {
|
||||
setDelayInput(String(globalSetting.request_delay_sec));
|
||||
setInitialized(true);
|
||||
}
|
||||
}, [globalSetting, initialized]);
|
||||
|
||||
return (
|
||||
<section className="scraper-section scraper-section--highlight">
|
||||
<h2>Глобальная задержка запросов (нижний порог)</h2>
|
||||
<p className="scraper-hint">
|
||||
Глобальный делэй — нижний порог для всех скрапперов (max с per-source
|
||||
defaults). Если 0 — только per-source defaults. Изменение вступает в
|
||||
силу немедленно без перезапуска процесса. Для per-source задержек
|
||||
зайдите на страницу конкретного скраппера.
|
||||
</p>
|
||||
|
||||
{settingsQ.isPending && (
|
||||
<p className="scraper-hint">Загрузка настроек…</p>
|
||||
)}
|
||||
{settingsQ.error && (
|
||||
<p className="scraper-result scraper-result--error">
|
||||
Ошибка загрузки: {settingsQ.error.message}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{globalSetting && (
|
||||
<div className="schedule-status" style={{ marginBottom: "12px" }}>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Текущий global delay:</span>
|
||||
<span>
|
||||
<strong>{globalSetting.request_delay_sec} сек</strong>
|
||||
</span>
|
||||
</div>
|
||||
{globalSetting.updated_at && (
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Обновлено:</span>
|
||||
<span>{formatTime(globalSetting.updated_at)}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Per-source read-only table */}
|
||||
{settingsQ.data && settingsQ.data.length > 0 && (
|
||||
<div style={{ marginBottom: "16px" }}>
|
||||
<p
|
||||
style={{
|
||||
fontSize: "0.85rem",
|
||||
fontWeight: 600,
|
||||
marginBottom: "6px",
|
||||
}}
|
||||
>
|
||||
Все per-source задержки (read-only здесь, редактировать на
|
||||
страницах провайдеров):
|
||||
</p>
|
||||
<table className="runs-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Source</th>
|
||||
<th>Delay (сек)</th>
|
||||
<th>Описание</th>
|
||||
<th>Обновлено</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{settingsQ.data.map((s) => (
|
||||
<tr
|
||||
key={s.source}
|
||||
className={
|
||||
s.source === "global"
|
||||
? "run-row run-row--running"
|
||||
: "run-row"
|
||||
}
|
||||
>
|
||||
<td>
|
||||
<code>{s.source}</code>
|
||||
</td>
|
||||
<td>{s.request_delay_sec.toFixed(1)}</td>
|
||||
<td
|
||||
style={{
|
||||
fontSize: "0.8rem",
|
||||
color: "var(--muted, #5b6066)",
|
||||
}}
|
||||
>
|
||||
{s.description ?? "—"}
|
||||
</td>
|
||||
<td style={{ fontSize: "0.8rem" }}>
|
||||
{formatTime(s.updated_at)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
updateMut.mutate({
|
||||
source: "global",
|
||||
request_delay_sec: parseFloat(delayInput),
|
||||
});
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
Global delay (сек, 0–30)
|
||||
<input
|
||||
type="number"
|
||||
step={0.5}
|
||||
min={0}
|
||||
max={30}
|
||||
value={delayInput}
|
||||
onChange={(e) => setDelayInput(e.target.value)}
|
||||
/>
|
||||
</label>
|
||||
<input
|
||||
type="range"
|
||||
min={0}
|
||||
max={30}
|
||||
step={0.5}
|
||||
value={delayInput}
|
||||
onChange={(e) => setDelayInput(e.target.value)}
|
||||
style={{ width: "100%", gridColumn: "1 / -1" }}
|
||||
/>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={updateMut.isPending}
|
||||
style={{ gridColumn: "1 / -1" }}
|
||||
>
|
||||
{updateMut.isPending ? "Сохраняем…" : "Сохранить global delay"}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
{updateMut.error && (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
Ошибка: {updateMut.error.message}
|
||||
</div>
|
||||
)}
|
||||
{updateMut.isSuccess && updateMut.data && (
|
||||
<div className="scraper-result">
|
||||
Global delay обновлён:{" "}
|
||||
<strong>
|
||||
{Number(updateMut.data.request_delay_sec).toFixed(1)} сек
|
||||
</strong>{" "}
|
||||
(обновлено {formatTime(updateMut.data.updated_at)})
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Schedules summary table ────────────────────────────────────────────────
|
||||
|
||||
const SCHEDULE_SOURCE_LABELS: Record<string, string> = {
|
||||
avito_city_sweep: "Avito city sweep",
|
||||
cian_city_sweep: "Cian city sweep",
|
||||
yandex_city_sweep: "Yandex city sweep",
|
||||
};
|
||||
|
||||
const SCHEDULE_SOURCES: ScheduleSourceKey[] = [
|
||||
"avito_city_sweep",
|
||||
"cian_city_sweep",
|
||||
"yandex_city_sweep",
|
||||
];
|
||||
|
||||
function SchedulesSummary() {
|
||||
const schedulesQ = useSchedules();
|
||||
const updateMut = useUpdateSchedule();
|
||||
|
||||
function handleToggle(schedule: ScheduleConfig) {
|
||||
updateMut.mutate({
|
||||
source: schedule.source as ScheduleSourceKey,
|
||||
payload: {
|
||||
enabled: !schedule.enabled,
|
||||
window_start_hour: schedule.window_start_hour,
|
||||
window_end_hour: schedule.window_end_hour,
|
||||
default_params: schedule.default_params as Record<string, unknown>,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Build ordered rows from known sources + any extras
|
||||
const knownRows = SCHEDULE_SOURCES.map((src) =>
|
||||
schedulesQ.data?.find((s) => s.source === src),
|
||||
).filter((s): s is ScheduleConfig => s !== undefined);
|
||||
|
||||
const extraRows =
|
||||
schedulesQ.data?.filter(
|
||||
(s) => !SCHEDULE_SOURCES.includes(s.source as ScheduleSourceKey),
|
||||
) ?? [];
|
||||
|
||||
const rows = [...knownRows, ...extraRows];
|
||||
|
||||
return (
|
||||
<section className="scraper-section">
|
||||
<h2>Сводная таблица расписаний</h2>
|
||||
<p className="scraper-hint">
|
||||
Быстрое включение/отключение всех cron-расписаний. Параметры и окна
|
||||
настраиваются на страницах провайдеров. Автообновление каждые 30 сек.
|
||||
</p>
|
||||
|
||||
{schedulesQ.isPending && <p className="scraper-hint">Загрузка…</p>}
|
||||
{schedulesQ.error && (
|
||||
<p className="scraper-result scraper-result--error">
|
||||
Ошибка загрузки: {schedulesQ.error.message}
|
||||
</p>
|
||||
)}
|
||||
|
||||
{rows.length > 0 && (
|
||||
<table className="runs-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Source</th>
|
||||
<th>Статус</th>
|
||||
<th>Окно (МСК)</th>
|
||||
<th>Следующий запуск</th>
|
||||
<th>Последний запуск</th>
|
||||
<th>Последний run</th>
|
||||
<th>Вкл/выкл</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{rows.map((s) => {
|
||||
const mskStart = utcToMsk(s.window_start_hour);
|
||||
const mskEnd = utcToMsk(s.window_end_hour);
|
||||
const label =
|
||||
SCHEDULE_SOURCE_LABELS[s.source] ?? s.source;
|
||||
return (
|
||||
<tr key={s.source} className="run-row">
|
||||
<td>
|
||||
<strong>{label}</strong>
|
||||
</td>
|
||||
<td>
|
||||
<span
|
||||
className={`run-badge run-badge--${s.enabled ? "running" : "cancelled"}`}
|
||||
>
|
||||
{s.enabled ? "включено" : "отключено"}
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
{String(mskStart).padStart(2, "0")}:00–
|
||||
{String(mskEnd).padStart(2, "0")}:00 МСК
|
||||
</td>
|
||||
<td>{formatTime(s.next_run_at)}</td>
|
||||
<td>{formatTime(s.last_run_at)}</td>
|
||||
<td>
|
||||
{s.last_run_id ? (
|
||||
<span className="run-muted">#{s.last_run_id}</span>
|
||||
) : (
|
||||
"—"
|
||||
)}
|
||||
</td>
|
||||
<td>
|
||||
<button
|
||||
type="button"
|
||||
className="cancel-btn"
|
||||
disabled={updateMut.isPending}
|
||||
onClick={() => handleToggle(s)}
|
||||
style={
|
||||
s.enabled
|
||||
? undefined
|
||||
: { background: "var(--success, #27ae60)" }
|
||||
}
|
||||
>
|
||||
{s.enabled ? "Отключить" : "Включить"}
|
||||
</button>
|
||||
</td>
|
||||
</tr>
|
||||
);
|
||||
})}
|
||||
</tbody>
|
||||
</table>
|
||||
)}
|
||||
|
||||
{rows.length === 0 && !schedulesQ.isPending && !schedulesQ.error && (
|
||||
<p className="scraper-hint">Расписания не найдены. Они создаются автоматически при первом сохранении на странице провайдера.</p>
|
||||
)}
|
||||
|
||||
{updateMut.error && (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
Ошибка обновления: {updateMut.error.message}
|
||||
</div>
|
||||
)}
|
||||
{updateMut.isSuccess && (
|
||||
<div className="scraper-result">
|
||||
Расписание {translateStatus(
|
||||
updateMut.data.enabled ? "running" : "cancelled",
|
||||
)}.
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Provider cards ─────────────────────────────────────────────────────────
|
||||
|
||||
const PROVIDERS = [
|
||||
{
|
||||
key: "avito",
|
||||
name: "Avito",
|
||||
href: "/scrapers/avito",
|
||||
description:
|
||||
"City sweep, houses catalog, detail enrichment, IMV evaluation",
|
||||
},
|
||||
{
|
||||
key: "cian",
|
||||
name: "Cian",
|
||||
href: "/scrapers/cian",
|
||||
description: "City sweep, detail enrichment, новостройки ЖК, cookies",
|
||||
},
|
||||
{
|
||||
key: "yandex",
|
||||
name: "Yandex Realty",
|
||||
href: "/scrapers/yandex",
|
||||
description: "City sweep, detail, ЖК landing, valuation",
|
||||
},
|
||||
];
|
||||
|
||||
function ProviderCards() {
|
||||
return (
|
||||
<section className="scraper-section">
|
||||
<h2>Провайдеры</h2>
|
||||
<div
|
||||
style={{
|
||||
display: "grid",
|
||||
gridTemplateColumns: "repeat(auto-fit, minmax(240px, 1fr))",
|
||||
gap: "16px",
|
||||
marginTop: "12px",
|
||||
}}
|
||||
>
|
||||
{PROVIDERS.map((p) => (
|
||||
<a
|
||||
key={p.key}
|
||||
href={`${API_BASE_URL}${p.href}`}
|
||||
style={{
|
||||
display: "block",
|
||||
padding: "16px 20px",
|
||||
border: "1px solid var(--border, #e5e7eb)",
|
||||
borderRadius: 8,
|
||||
background: "var(--surface, #fff)",
|
||||
textDecoration: "none",
|
||||
color: "inherit",
|
||||
transition: "border-color 0.15s, box-shadow 0.15s",
|
||||
}}
|
||||
>
|
||||
<p
|
||||
style={{
|
||||
margin: "0 0 4px",
|
||||
fontWeight: 700,
|
||||
fontSize: "1rem",
|
||||
color: "var(--accent, #1d4ed8)",
|
||||
}}
|
||||
>
|
||||
{p.name}
|
||||
</p>
|
||||
<p
|
||||
style={{
|
||||
margin: 0,
|
||||
fontSize: "0.85rem",
|
||||
color: "var(--muted, #5b6066)",
|
||||
}}
|
||||
>
|
||||
{p.description}
|
||||
</p>
|
||||
</a>
|
||||
))}
|
||||
</div>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Page ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export default function ScrapersHubPage() {
|
||||
return (
|
||||
<>
|
||||
<Topbar active="scrapers" />
|
||||
<main className="page scraper-page">
|
||||
<h1 className="scraper-h1">Скрапперы — хаб</h1>
|
||||
<p className="scraper-subtitle">
|
||||
Центр управления всеми скрапперами: глобальные настройки задержки,
|
||||
сводная таблица расписаний, ссылки на admin-триггеры провайдеров.
|
||||
</p>
|
||||
|
||||
<ProviderCards />
|
||||
<GlobalDelaySetting />
|
||||
<SchedulesSummary />
|
||||
</main>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
|
@ -1,34 +1,16 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
import { useState } from "react";
|
||||
import { useMutation } from "@tanstack/react-query";
|
||||
|
||||
import "@/components/trade-in/trade-in.css";
|
||||
import { Topbar } from "@/components/trade-in/Topbar";
|
||||
import { apiFetch } from "@/lib/api";
|
||||
import {
|
||||
ResultPanel,
|
||||
ScraperPage,
|
||||
type ScraperPageConfig,
|
||||
} from "@/app/scrapers/_components/ScraperPage";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
interface ScraperSetting {
|
||||
source: string;
|
||||
request_delay_sec: number;
|
||||
description: string | null;
|
||||
updated_at: string | null;
|
||||
}
|
||||
|
||||
interface ScraperSettingUpdate {
|
||||
source: string;
|
||||
request_delay_sec: number;
|
||||
}
|
||||
|
||||
interface ScrapeAroundInput {
|
||||
lat: number;
|
||||
lon: number;
|
||||
radius_m: number;
|
||||
sources: string[];
|
||||
multi_room_yandex?: boolean;
|
||||
deep_yandex?: boolean;
|
||||
}
|
||||
// ── Yandex-specific types + hooks ──────────────────────────────────────────
|
||||
|
||||
interface ScrapeAroundResp {
|
||||
total_fetched: number;
|
||||
|
|
@ -75,43 +57,19 @@ interface YandexValuationTriggerResp {
|
|||
history_items_count: number;
|
||||
}
|
||||
|
||||
// ── Hooks ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function useScraperSettings() {
|
||||
return useQuery<ScraperSetting[]>({
|
||||
queryKey: ["scraper-settings"],
|
||||
queryFn: () =>
|
||||
apiFetch<{ settings: ScraperSetting[] }>("/api/v1/admin/scraper-settings").then(
|
||||
(r) =>
|
||||
(r.settings ?? []).map((s) => ({
|
||||
...s,
|
||||
request_delay_sec: Number(s.request_delay_sec),
|
||||
})),
|
||||
),
|
||||
refetchInterval: 30_000,
|
||||
});
|
||||
}
|
||||
|
||||
function useUpdateScraperSetting() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation<
|
||||
ScraperSetting,
|
||||
Error,
|
||||
{ source: string; payload: Omit<ScraperSettingUpdate, "source"> }
|
||||
>({
|
||||
mutationFn: ({ source, payload }) =>
|
||||
apiFetch<ScraperSetting>(`/api/v1/admin/scraper-settings/${source}`, {
|
||||
method: "PUT",
|
||||
body: JSON.stringify({ source, ...payload }),
|
||||
}),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ["scraper-settings"] });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function useScrapeAround() {
|
||||
return useMutation<ScrapeAroundResp, Error, ScrapeAroundInput>({
|
||||
return useMutation<
|
||||
ScrapeAroundResp,
|
||||
Error,
|
||||
{
|
||||
lat: number;
|
||||
lon: number;
|
||||
radius_m: number;
|
||||
sources: string[];
|
||||
multi_room_yandex?: boolean;
|
||||
deep_yandex?: boolean;
|
||||
}
|
||||
>({
|
||||
mutationFn: (input) =>
|
||||
apiFetch<ScrapeAroundResp>("/api/v1/admin/scrape", {
|
||||
method: "POST",
|
||||
|
|
@ -150,7 +108,12 @@ function useScrapeYandexValuation() {
|
|||
return useMutation<
|
||||
YandexValuationTriggerResp,
|
||||
Error,
|
||||
{ address: string; offer_category: string; offer_type: string; page: number }
|
||||
{
|
||||
address: string;
|
||||
offer_category: string;
|
||||
offer_type: string;
|
||||
page: number;
|
||||
}
|
||||
>({
|
||||
mutationFn: ({ address, offer_category, offer_type, page }) => {
|
||||
const qs = new URLSearchParams({
|
||||
|
|
@ -167,69 +130,9 @@ function useScrapeYandexValuation() {
|
|||
});
|
||||
}
|
||||
|
||||
// ── Result panel ───────────────────────────────────────────────────────────
|
||||
|
||||
interface ResultPanelProps<TData> {
|
||||
mut: {
|
||||
data?: TData;
|
||||
error: Error | null;
|
||||
isPending: boolean;
|
||||
isSuccess: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
function ResultPanel<TData>({ mut }: ResultPanelProps<TData>) {
|
||||
if (mut.error) {
|
||||
return (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
<strong>Ошибка:</strong> {mut.error.message}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (mut.isSuccess && mut.data) {
|
||||
return (
|
||||
<div className="scraper-result">
|
||||
<pre>{JSON.stringify(mut.data, null, 2)}</pre>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
function formatTime(iso: string | null): string {
|
||||
if (!iso) return "—";
|
||||
try {
|
||||
return new Date(iso).toLocaleString("ru-RU", {
|
||||
day: "2-digit",
|
||||
month: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
});
|
||||
} catch {
|
||||
return iso;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Page ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export default function YandexScraperPage() {
|
||||
// Settings
|
||||
const settingsQ = useScraperSettings();
|
||||
const updateSettingMut = useUpdateScraperSetting();
|
||||
const [delayInput, setDelayInput] = useState<string>("5.0");
|
||||
const delayInitializedRef = useRef(false);
|
||||
|
||||
useEffect(() => {
|
||||
const yandex = settingsQ.data?.find((s) => s.source === "yandex");
|
||||
if (yandex && !delayInitializedRef.current) {
|
||||
setDelayInput(String(yandex.request_delay_sec));
|
||||
delayInitializedRef.current = true;
|
||||
}
|
||||
}, [settingsQ.data]);
|
||||
// ── Yandex extra triggers ──────────────────────────────────────────────────
|
||||
|
||||
function YandexExtraTriggers() {
|
||||
// Around
|
||||
const [lat, setLat] = useState("56.8400");
|
||||
const [lon, setLon] = useState("60.6050");
|
||||
|
|
@ -259,317 +162,263 @@ export default function YandexScraperPage() {
|
|||
const [valPage, setValPage] = useState("1");
|
||||
const valMut = useScrapeYandexValuation();
|
||||
|
||||
const yandexSetting = settingsQ.data?.find((s) => s.source === "yandex");
|
||||
|
||||
return (
|
||||
<>
|
||||
<Topbar active="yandex" />
|
||||
<main className="page scraper-page">
|
||||
<h1 className="scraper-h1">Yandex скрапер — admin триггер</h1>
|
||||
<p className="scraper-subtitle">
|
||||
Ручной запуск sub-pipeline'ов YandexRealtyScraper v1 + глобальная настройка
|
||||
задержки между запросами.
|
||||
{/* 1. Around point */}
|
||||
<div className="scraper-section">
|
||||
<h3>1. Search around (общий /admin/scrape)</h3>
|
||||
<p className="scraper-hint">
|
||||
Yandex SERP path-based vtorichka — lat/lon/radius логируется, scraper
|
||||
берёт{" "}
|
||||
<code>/{ekaterinburg}/kupit/kvartira/vtorichniy-rynok/</code>.
|
||||
multi-room split не реализован для DOM-парсера. Deep пробует pages ×
|
||||
sorts комбинации.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
aroundMut.mutate({
|
||||
lat: parseFloat(lat),
|
||||
lon: parseFloat(lon),
|
||||
radius_m: parseInt(radius, 10),
|
||||
sources: ["yandex"],
|
||||
multi_room_yandex: multiRoom,
|
||||
deep_yandex: deep,
|
||||
});
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
Lat
|
||||
<input
|
||||
value={lat}
|
||||
onChange={(e) => setLat(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Lon
|
||||
<input
|
||||
value={lon}
|
||||
onChange={(e) => setLon(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Radius (м)
|
||||
<input
|
||||
type="number"
|
||||
min={100}
|
||||
max={10000}
|
||||
step={100}
|
||||
value={radius}
|
||||
onChange={(e) => setRadius(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={multiRoom}
|
||||
onChange={(e) => setMultiRoom(e.target.checked)}
|
||||
/>
|
||||
multi_room_yandex (5 rooms split)
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={deep}
|
||||
onChange={(e) => setDeep(e.target.checked)}
|
||||
/>
|
||||
deep_yandex (5 × 3 × 2 = 30 requests)
|
||||
</label>
|
||||
<button type="submit" disabled={aroundMut.isPending}>
|
||||
{aroundMut.isPending ? "Скрейпим…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={aroundMut} />
|
||||
</div>
|
||||
|
||||
{/* 0. Global delay */}
|
||||
<section className="scraper-section scraper-section--highlight">
|
||||
<h2>Глобальная задержка запросов</h2>
|
||||
<p className="scraper-hint">
|
||||
Настройка применяется ко ВСЕМ Yandex-скрейперам (SERP, detail, ЖК, valuation)
|
||||
при создании следующего инстанса (in-process cache 60s). Меняется без рестарта
|
||||
backend.
|
||||
</p>
|
||||
{/* 2. Detail */}
|
||||
<div className="scraper-section">
|
||||
<h3>2. Detail page enrichment</h3>
|
||||
<p className="scraper-hint">
|
||||
Product JSON-LD price (exact int) + DOM-секции
|
||||
description/agency/metro/photos.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
detailMut.mutate({ offer_url: detailUrl });
|
||||
}}
|
||||
>
|
||||
<label className="full">
|
||||
offer_url
|
||||
<input
|
||||
value={detailUrl}
|
||||
onChange={(e) => setDetailUrl(e.target.value)}
|
||||
required
|
||||
placeholder="https://realty.yandex.ru/offer/..."
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={detailMut.isPending}>
|
||||
{detailMut.isPending ? "Парсим…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={detailMut} />
|
||||
</div>
|
||||
|
||||
{settingsQ.isPending && <p className="scraper-hint">Загрузка настроек…</p>}
|
||||
{settingsQ.error && (
|
||||
<p className="scraper-result scraper-result--error">
|
||||
Ошибка загрузки настроек: {settingsQ.error.message}
|
||||
</p>
|
||||
)}
|
||||
{/* 3. Newbuilding */}
|
||||
<div className="scraper-section">
|
||||
<h3>3. ЖК (Newbuilding) landing</h3>
|
||||
<p className="scraper-hint">
|
||||
URL /{city}/kupit/novostrojka/{slug}-{id}/ — извлекает coords
|
||||
inline, rating, text_reviews_count, developer_name.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
nbMut.mutate({ slug: nbSlug, id: nbId, city: nbCity });
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
slug
|
||||
<input
|
||||
value={nbSlug}
|
||||
onChange={(e) => setNbSlug(e.target.value)}
|
||||
required
|
||||
placeholder="tatlin"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
id
|
||||
<input
|
||||
value={nbId}
|
||||
onChange={(e) => setNbId(e.target.value)}
|
||||
required
|
||||
placeholder="1592987"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
city
|
||||
<input
|
||||
value={nbCity}
|
||||
onChange={(e) => setNbCity(e.target.value)}
|
||||
required
|
||||
placeholder="ekaterinburg"
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={nbMut.isPending}>
|
||||
{nbMut.isPending ? "Парсим…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={nbMut} />
|
||||
</div>
|
||||
|
||||
{yandexSetting && (
|
||||
<div className="schedule-status" style={{ marginBottom: "12px" }}>
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Текущая задержка:</span>
|
||||
<span>
|
||||
<strong>{yandexSetting.request_delay_sec} сек</strong>
|
||||
</span>
|
||||
</div>
|
||||
{yandexSetting.updated_at && (
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Обновлено:</span>
|
||||
<span>{formatTime(yandexSetting.updated_at)}</span>
|
||||
</div>
|
||||
)}
|
||||
{yandexSetting.description && (
|
||||
<div className="schedule-status__row">
|
||||
<span className="schedule-status__label">Описание:</span>
|
||||
<span className="run-muted">{yandexSetting.description}</span>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
updateSettingMut.mutate({
|
||||
source: "yandex",
|
||||
payload: { request_delay_sec: parseFloat(delayInput) },
|
||||
});
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
Задержка (сек, 1–60)
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={60}
|
||||
step={0.5}
|
||||
value={delayInput}
|
||||
onChange={(e) => setDelayInput(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={updateSettingMut.isPending}>
|
||||
{updateSettingMut.isPending ? "Сохраняем…" : "Сохранить"}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
{updateSettingMut.error && (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
<strong>Ошибка:</strong> {updateSettingMut.error.message}
|
||||
</div>
|
||||
)}
|
||||
{updateSettingMut.isSuccess && updateSettingMut.data && (
|
||||
<div className="scraper-result">
|
||||
Сохранено. Новая задержка:{" "}
|
||||
<strong>{updateSettingMut.data.request_delay_sec} сек</strong>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
{/* 1. Around point */}
|
||||
<section className="scraper-section">
|
||||
<h2>1. Search around (общий /admin/scrape)</h2>
|
||||
<p className="scraper-hint">
|
||||
Yandex SERP path-based vtorichka — lat/lon/radius логируется, scraper берёт{" "}
|
||||
<code>/{ekaterinburg}/kupit/kvartira/vtorichniy-rynok/</code>.
|
||||
multi-room split не реализован для DOM-парсера (запросы будут одинаковыми).
|
||||
Deep пробует pages × sorts комбинации.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
aroundMut.mutate({
|
||||
lat: parseFloat(lat),
|
||||
lon: parseFloat(lon),
|
||||
radius_m: parseInt(radius, 10),
|
||||
sources: ["yandex"],
|
||||
multi_room_yandex: multiRoom,
|
||||
deep_yandex: deep,
|
||||
});
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
Lat
|
||||
<input
|
||||
value={lat}
|
||||
onChange={(e) => setLat(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Lon
|
||||
<input
|
||||
value={lon}
|
||||
onChange={(e) => setLon(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Radius (м)
|
||||
<input
|
||||
type="number"
|
||||
min={100}
|
||||
max={10000}
|
||||
step={100}
|
||||
value={radius}
|
||||
onChange={(e) => setRadius(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={multiRoom}
|
||||
onChange={(e) => setMultiRoom(e.target.checked)}
|
||||
/>
|
||||
multi_room_yandex (5 rooms split)
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={deep}
|
||||
onChange={(e) => setDeep(e.target.checked)}
|
||||
/>
|
||||
deep_yandex (5 × 3 × 2 = 30 requests)
|
||||
</label>
|
||||
<button type="submit" disabled={aroundMut.isPending}>
|
||||
{aroundMut.isPending ? "Скрейпим…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={aroundMut} />
|
||||
</section>
|
||||
|
||||
{/* 2. Detail */}
|
||||
<section className="scraper-section">
|
||||
<h2>2. Detail page enrichment</h2>
|
||||
<p className="scraper-hint">
|
||||
Product JSON-LD price (exact int) + DOM-секции description/agency/metro/photos.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
detailMut.mutate({ offer_url: detailUrl });
|
||||
}}
|
||||
>
|
||||
<label className="full">
|
||||
offer_url
|
||||
<input
|
||||
value={detailUrl}
|
||||
onChange={(e) => setDetailUrl(e.target.value)}
|
||||
required
|
||||
placeholder="https://realty.yandex.ru/offer/..."
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={detailMut.isPending}>
|
||||
{detailMut.isPending ? "Парсим…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={detailMut} />
|
||||
</section>
|
||||
|
||||
{/* 3. ЖК Newbuilding */}
|
||||
<section className="scraper-section">
|
||||
<h2>3. ЖК (Newbuilding) landing</h2>
|
||||
<p className="scraper-hint">
|
||||
URL /{city}/kupit/novostrojka/{slug}-{id}/ — извлекает coords inline,
|
||||
rating, text_reviews_count, developer_name.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
nbMut.mutate({ slug: nbSlug, id: nbId, city: nbCity });
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
slug
|
||||
<input
|
||||
value={nbSlug}
|
||||
onChange={(e) => setNbSlug(e.target.value)}
|
||||
required
|
||||
placeholder="tatlin"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
id
|
||||
<input
|
||||
value={nbId}
|
||||
onChange={(e) => setNbId(e.target.value)}
|
||||
required
|
||||
placeholder="1592987"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
city
|
||||
<input
|
||||
value={nbCity}
|
||||
onChange={(e) => setNbCity(e.target.value)}
|
||||
required
|
||||
placeholder="ekaterinburg"
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={nbMut.isPending}>
|
||||
{nbMut.isPending ? "Парсим…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={nbMut} />
|
||||
</section>
|
||||
|
||||
{/* 4. Valuation */}
|
||||
<section className="scraper-section">
|
||||
<h2>4. Valuation (anonymous history)</h2>
|
||||
<p className="scraper-hint">
|
||||
Anonymous GET /otsenka-kvartiry-po-adresu-onlayn/ — house meta + до 30 history
|
||||
items на page.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
valMut.mutate({
|
||||
address: valAddress,
|
||||
offer_category: valCategory,
|
||||
offer_type: valType,
|
||||
page: parseInt(valPage, 10),
|
||||
});
|
||||
}}
|
||||
>
|
||||
<label className="full">
|
||||
Адрес
|
||||
<input
|
||||
value={valAddress}
|
||||
onChange={(e) => setValAddress(e.target.value)}
|
||||
required
|
||||
placeholder="Свердловская область, Екатеринбург, улица Учителей, 18"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
offer_category
|
||||
<select
|
||||
value={valCategory}
|
||||
onChange={(e) => setValCategory(e.target.value)}
|
||||
>
|
||||
<option value="APARTMENT">APARTMENT</option>
|
||||
<option value="HOUSE">HOUSE</option>
|
||||
<option value="GARAGE">GARAGE</option>
|
||||
<option value="COMMERCIAL">COMMERCIAL</option>
|
||||
<option value="ROOM">ROOM</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
offer_type
|
||||
<select
|
||||
value={valType}
|
||||
onChange={(e) => setValType(e.target.value)}
|
||||
>
|
||||
<option value="SELL">SELL</option>
|
||||
<option value="RENT">RENT</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
page
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={100}
|
||||
value={valPage}
|
||||
onChange={(e) => setValPage(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={valMut.isPending}>
|
||||
{valMut.isPending ? "Запрашиваем…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={valMut} />
|
||||
</section>
|
||||
</main>
|
||||
{/* 4. Valuation */}
|
||||
<div className="scraper-section">
|
||||
<h3>4. Valuation (anonymous history)</h3>
|
||||
<p className="scraper-hint">
|
||||
Anonymous GET /otsenka-kvartiry-po-adresu-onlayn/ — house meta + до
|
||||
30 history items на page.
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
valMut.mutate({
|
||||
address: valAddress,
|
||||
offer_category: valCategory,
|
||||
offer_type: valType,
|
||||
page: parseInt(valPage, 10),
|
||||
});
|
||||
}}
|
||||
>
|
||||
<label className="full">
|
||||
Адрес
|
||||
<input
|
||||
value={valAddress}
|
||||
onChange={(e) => setValAddress(e.target.value)}
|
||||
required
|
||||
placeholder="Свердловская область, Екатеринбург, улица Учителей, 18"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
offer_category
|
||||
<select
|
||||
value={valCategory}
|
||||
onChange={(e) => setValCategory(e.target.value)}
|
||||
>
|
||||
<option value="APARTMENT">APARTMENT</option>
|
||||
<option value="HOUSE">HOUSE</option>
|
||||
<option value="GARAGE">GARAGE</option>
|
||||
<option value="COMMERCIAL">COMMERCIAL</option>
|
||||
<option value="ROOM">ROOM</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
offer_type
|
||||
<select
|
||||
value={valType}
|
||||
onChange={(e) => setValType(e.target.value)}
|
||||
>
|
||||
<option value="SELL">SELL</option>
|
||||
<option value="RENT">RENT</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
page
|
||||
<input
|
||||
type="number"
|
||||
min={1}
|
||||
max={100}
|
||||
value={valPage}
|
||||
onChange={(e) => setValPage(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={valMut.isPending}>
|
||||
{valMut.isPending ? "Запрашиваем…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={valMut} />
|
||||
</div>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Page config + export ───────────────────────────────────────────────────
|
||||
|
||||
const YANDEX_CONFIG: ScraperPageConfig = {
|
||||
source: "yandex",
|
||||
activeTab: "yandex",
|
||||
displayName: "Yandex скрапер",
|
||||
subtitle:
|
||||
"Ручной запуск sub-pipeline'ов YandexRealtyScraper v1. Для debug и теста после изменений.",
|
||||
scheduleSource: "yandex_city_sweep",
|
||||
paramConfig: {
|
||||
hasDetailParams: false,
|
||||
hasEnrichAddress: true,
|
||||
defaults: {
|
||||
pages_per_anchor: 3,
|
||||
request_delay_sec: 5,
|
||||
radius_m: 1500,
|
||||
enrich_address: false,
|
||||
},
|
||||
},
|
||||
delayMin: 1,
|
||||
delayMax: 60,
|
||||
};
|
||||
|
||||
export default function YandexScraperPage() {
|
||||
return (
|
||||
<ScraperPage
|
||||
config={{
|
||||
...YANDEX_CONFIG,
|
||||
extraTriggers: <YandexExtraTriggers />,
|
||||
}}
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -9,10 +9,11 @@ import { safeUrl } from "@/lib/safeUrl";
|
|||
import { useBrand } from "@/lib/useBrand";
|
||||
import { useMe } from "@/lib/useMe";
|
||||
|
||||
type ActiveTab =
|
||||
export type ActiveTab =
|
||||
| "estimate"
|
||||
| "history"
|
||||
| "cache"
|
||||
| "scrapers"
|
||||
| "avito"
|
||||
| "cian"
|
||||
| "yandex"
|
||||
|
|
@ -48,6 +49,12 @@ const NAV_ITEMS: Array<{
|
|||
{ key: "history", href: "/history", scopePath: "/trade-in/history", label: "История" },
|
||||
{ key: "cache", href: "/cache", scopePath: "/trade-in/cache", label: "Кэш" },
|
||||
// Скраперы — admin-only UI. Маппим на admin-deny path, чтобы pilot их не видел.
|
||||
{
|
||||
key: "scrapers",
|
||||
href: "/scrapers",
|
||||
scopePath: "/trade-in/api/v1/admin/scrapers",
|
||||
label: "Скрапперы",
|
||||
},
|
||||
{
|
||||
key: "avito",
|
||||
href: "/scrapers/avito",
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue