feat(tradein-ui): Cian scraper admin page + global request delay control (#486)
This commit is contained in:
parent
b21d7c7c85
commit
ebfd29a0a6
1 changed files with 435 additions and 8 deletions
|
|
@ -1,24 +1,451 @@
|
|||
"use client";
|
||||
|
||||
import { useEffect, useState } from "react";
|
||||
import Link from "next/link";
|
||||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||||
|
||||
import "@/components/trade-in/trade-in.css";
|
||||
import { Topbar } from "@/components/trade-in/Topbar";
|
||||
import { apiFetch } from "@/lib/api";
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
|
||||
interface ScrapeAroundResp {
|
||||
total_fetched: number;
|
||||
total_inserted: number;
|
||||
total_updated: number;
|
||||
by_source: Array<{
|
||||
source: string;
|
||||
fetched: number;
|
||||
inserted: number;
|
||||
updated: number;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface TestAuthResp {
|
||||
authenticated: boolean;
|
||||
userId: number | null;
|
||||
reason: string | null;
|
||||
}
|
||||
|
||||
interface ScraperSetting {
|
||||
source: string;
|
||||
request_delay_sec: number;
|
||||
description: string | null;
|
||||
updated_at: string | null;
|
||||
}
|
||||
|
||||
interface ScraperSettingUpdate {
|
||||
request_delay_sec: number;
|
||||
}
|
||||
|
||||
// ── Hooks ──────────────────────────────────────────────────────────────────
|
||||
|
||||
function useScrapeAround() {
|
||||
return useMutation<
|
||||
ScrapeAroundResp,
|
||||
Error,
|
||||
{ lat: number; lon: number; radius_m: number; multi_room_cian: boolean }
|
||||
>({
|
||||
mutationFn: (input) =>
|
||||
apiFetch<ScrapeAroundResp>("/api/v1/admin/scrape", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ ...input, sources: ["cian"] }),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
function useTestAuth() {
|
||||
return useQuery<TestAuthResp>({
|
||||
queryKey: ["cian-test-auth"],
|
||||
queryFn: () => apiFetch<TestAuthResp>("/api/v1/admin/scrape/cian/test-auth"),
|
||||
staleTime: 30_000,
|
||||
refetchInterval: 60_000,
|
||||
});
|
||||
}
|
||||
|
||||
function useScraperSettings() {
|
||||
return useQuery<ScraperSetting[]>({
|
||||
queryKey: ["scraper-settings"],
|
||||
queryFn: () =>
|
||||
apiFetch<ScraperSetting[]>("/api/v1/admin/scraper-settings"),
|
||||
staleTime: 30_000,
|
||||
});
|
||||
}
|
||||
|
||||
function useUpdateScraperSetting() {
|
||||
const qc = useQueryClient();
|
||||
return useMutation<
|
||||
ScraperSetting,
|
||||
Error,
|
||||
{ source: string; payload: ScraperSettingUpdate }
|
||||
>({
|
||||
mutationFn: ({ source, payload }) =>
|
||||
apiFetch<ScraperSetting>(`/api/v1/admin/scraper-settings/${source}`, {
|
||||
method: "PUT",
|
||||
body: JSON.stringify(payload),
|
||||
}),
|
||||
onSuccess: () => {
|
||||
qc.invalidateQueries({ queryKey: ["scraper-settings"] });
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ── Result panel ───────────────────────────────────────────────────────────
|
||||
|
||||
interface ResultPanelProps<TData> {
|
||||
mut: {
|
||||
data?: TData;
|
||||
error: Error | null;
|
||||
isPending: boolean;
|
||||
isSuccess: boolean;
|
||||
};
|
||||
}
|
||||
|
||||
function ResultPanel<TData>({ mut }: ResultPanelProps<TData>) {
|
||||
if (mut.error) {
|
||||
return (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
<strong>Ошибка:</strong> {mut.error.message}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
if (mut.isSuccess && mut.data) {
|
||||
return (
|
||||
<div className="scraper-result">
|
||||
<pre>{JSON.stringify(mut.data, null, 2)}</pre>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// ── Page ───────────────────────────────────────────────────────────────────
|
||||
|
||||
export default function CianScraperPage() {
|
||||
// Auth status
|
||||
const authQ = useTestAuth();
|
||||
|
||||
// 1. Search around
|
||||
const [lat, setLat] = useState("56.8400");
|
||||
const [lon, setLon] = useState("60.6050");
|
||||
const [radius, setRadius] = useState("1500");
|
||||
const [multiRoom, setMultiRoom] = useState(false);
|
||||
const aroundMut = useScrapeAround();
|
||||
|
||||
// Global delay
|
||||
const settingsQ = useScraperSettings();
|
||||
const updateSettingMut = useUpdateScraperSetting();
|
||||
|
||||
const [globalDelay, setGlobalDelay] = useState<number>(5);
|
||||
|
||||
useEffect(() => {
|
||||
const global = settingsQ.data?.find((s) => s.source === "global");
|
||||
if (global !== undefined) {
|
||||
setGlobalDelay(global.request_delay_sec);
|
||||
}
|
||||
}, [settingsQ.data]);
|
||||
|
||||
return (
|
||||
<>
|
||||
<Topbar active="cian" />
|
||||
<main className="page scraper-page">
|
||||
<h1 className="scraper-h1">Cian скрапер</h1>
|
||||
<p className="scraper-subtitle">В разработке.</p>
|
||||
<div className="scraper-section">
|
||||
<p style={{ color: "var(--muted, #5b6066)" }}>
|
||||
Cian SERP refactor merged (PR #450), но admin UI ещё не подключён.
|
||||
Используй cron-scrape.sh или прямой curl к{" "}
|
||||
<code>POST /api/v1/admin/scrape</code> с{" "}
|
||||
<code>sources: ["cian"]</code>.
|
||||
<h1 className="scraper-h1">Cian скрапер — admin триггер</h1>
|
||||
<p className="scraper-subtitle">
|
||||
Ручной запуск sub-pipeline'ов CianScraper. Для debug и теста после
|
||||
изменений.
|
||||
</p>
|
||||
|
||||
{/* Cookies callout */}
|
||||
<div
|
||||
className="scraper-section"
|
||||
style={{
|
||||
background: "var(--surface-2, #f0f4ff)",
|
||||
borderLeft: "4px solid var(--accent, #3b6cff)",
|
||||
padding: "12px 16px",
|
||||
}}
|
||||
>
|
||||
<p style={{ margin: 0, fontSize: "0.9rem" }}>
|
||||
<strong>Stage 7: Cian Valuation</strong> требует загруженных cookies.{" "}
|
||||
<Link href="/trade-in/scrapers/cian-cookies">
|
||||
Настроить cookies →
|
||||
</Link>
|
||||
</p>
|
||||
<p style={{ margin: "6px 0 0", fontSize: "0.85rem", color: "var(--muted, #5b6066)" }}>
|
||||
Статус сессии:{" "}
|
||||
{authQ.isPending && "проверяем…"}
|
||||
{authQ.error && (
|
||||
<span style={{ color: "var(--error, #c0392b)" }}>
|
||||
ошибка проверки
|
||||
</span>
|
||||
)}
|
||||
{authQ.data && (
|
||||
<span
|
||||
style={{
|
||||
color: authQ.data.authenticated
|
||||
? "var(--success, #27ae60)"
|
||||
: "var(--error, #c0392b)",
|
||||
fontWeight: 600,
|
||||
}}
|
||||
>
|
||||
{authQ.data.authenticated
|
||||
? `авторизован (userId: ${authQ.data.userId ?? "—"})`
|
||||
: `не авторизован — ${authQ.data.reason ?? "неизвестная причина"}`}
|
||||
</span>
|
||||
)}
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* 1. Search around point */}
|
||||
<section className="scraper-section">
|
||||
<h2>1. Search around (общий /admin/scrape)</h2>
|
||||
<p className="scraper-hint">
|
||||
Запускает Cian SERP pipeline для одного якоря: search → save_listings.
|
||||
Поддерживает multi_room режим (~4× лотов, отдельный запрос на каждое
|
||||
кол-во комнат).
|
||||
</p>
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
aroundMut.mutate({
|
||||
lat: parseFloat(lat),
|
||||
lon: parseFloat(lon),
|
||||
radius_m: parseInt(radius, 10),
|
||||
multi_room_cian: multiRoom,
|
||||
});
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
Lat
|
||||
<input
|
||||
value={lat}
|
||||
onChange={(e) => setLat(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Lon
|
||||
<input
|
||||
value={lon}
|
||||
onChange={(e) => setLon(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
Radius (м)
|
||||
<input
|
||||
value={radius}
|
||||
onChange={(e) => setRadius(e.target.value)}
|
||||
required
|
||||
/>
|
||||
</label>
|
||||
<label className="checkbox">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={multiRoom}
|
||||
onChange={(e) => setMultiRoom(e.target.checked)}
|
||||
/>
|
||||
multi_room_cian (4 запроса по комнатам)
|
||||
</label>
|
||||
<button type="submit" disabled={aroundMut.isPending}>
|
||||
{aroundMut.isPending ? "Скрейпим…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={aroundMut} />
|
||||
</section>
|
||||
|
||||
{/* 2. Scrape detail by URL — NOT YET IMPLEMENTED BACKEND */}
|
||||
<section className="scraper-section">
|
||||
<h2>2. Detail page enrichment</h2>
|
||||
<p className="scraper-hint">
|
||||
Парсинг конкретного объявления Cian по URL. Endpoint{" "}
|
||||
<code>POST /api/v1/admin/scrape/cian-detail</code> планируется в
|
||||
следующем PR. Пока используйте поиск по anchor (#1) для массового
|
||||
скрейпа.
|
||||
</p>
|
||||
<div
|
||||
className="scraper-result scraper-result--error"
|
||||
style={{ marginTop: 0 }}
|
||||
>
|
||||
Endpoint не реализован на backend. Запрос вернёт 404.
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* 3. Newbuilding — NOT YET IMPLEMENTED BACKEND */}
|
||||
<section className="scraper-section">
|
||||
<h2>3. Новостройки (ЖК scrape)</h2>
|
||||
<p className="scraper-hint">
|
||||
Endpoint <code>POST /api/v1/admin/scrape/cian-newbuilding</code>{" "}
|
||||
планируется. Для Yandex Недвижимость аналог уже доступен на вкладке
|
||||
Yandex.
|
||||
</p>
|
||||
<div
|
||||
className="scraper-result scraper-result--error"
|
||||
style={{ marginTop: 0 }}
|
||||
>
|
||||
Endpoint не реализован на backend. Запрос вернёт 404.
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* 4. Valuation Calculator — NOT YET IMPLEMENTED BACKEND */}
|
||||
<section className="scraper-section">
|
||||
<h2>4. Valuation Calculator (оценка квартиры)</h2>
|
||||
<p className="scraper-hint">
|
||||
Endpoint <code>POST /api/v1/admin/scrape/cian-valuation</code>{" "}
|
||||
планируется. Требует загруженных cookies (Stage 7).{" "}
|
||||
<Link href="/trade-in/scrapers/cian-cookies">
|
||||
Настроить cookies →
|
||||
</Link>
|
||||
</p>
|
||||
<div
|
||||
className="scraper-result scraper-result--error"
|
||||
style={{ marginTop: 0 }}
|
||||
>
|
||||
Endpoint не реализован на backend. Запрос вернёт 404.
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* 5. Global Request Delay */}
|
||||
<section className="scraper-section scraper-section--highlight">
|
||||
<h2>5. Глобальный задержка запросов (Global Request Delay)</h2>
|
||||
<p className="scraper-hint">
|
||||
Глобальный делэй применяется ко всем scrapers (max с per-source
|
||||
defaults). 0 = только per-source defaults. Изменение вступает в силу
|
||||
немедленно — без перезапуска процесса.
|
||||
</p>
|
||||
|
||||
{/* Per-source read-only list */}
|
||||
{settingsQ.isPending && (
|
||||
<p className="scraper-hint">Загрузка настроек…</p>
|
||||
)}
|
||||
{settingsQ.error && (
|
||||
<p className="scraper-result scraper-result--error">
|
||||
Ошибка загрузки: {settingsQ.error.message}
|
||||
</p>
|
||||
)}
|
||||
{settingsQ.data && settingsQ.data.length > 0 && (
|
||||
<div style={{ marginBottom: "12px" }}>
|
||||
<p
|
||||
style={{
|
||||
fontSize: "0.85rem",
|
||||
fontWeight: 600,
|
||||
marginBottom: "4px",
|
||||
}}
|
||||
>
|
||||
Текущие per-source задержки (read-only):
|
||||
</p>
|
||||
<table className="runs-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Source</th>
|
||||
<th>Delay (сек)</th>
|
||||
<th>Описание</th>
|
||||
<th>Обновлено</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{settingsQ.data.map((s) => (
|
||||
<tr
|
||||
key={s.source}
|
||||
className={
|
||||
s.source === "global" ? "run-row run-row--running" : "run-row"
|
||||
}
|
||||
>
|
||||
<td>
|
||||
<code>{s.source}</code>
|
||||
</td>
|
||||
<td>{s.request_delay_sec.toFixed(1)}</td>
|
||||
<td
|
||||
style={{
|
||||
fontSize: "0.8rem",
|
||||
color: "var(--muted, #5b6066)",
|
||||
}}
|
||||
>
|
||||
{s.description ?? "—"}
|
||||
</td>
|
||||
<td style={{ fontSize: "0.8rem" }}>
|
||||
{formatTime(s.updated_at)}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Global delay edit form */}
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
updateSettingMut.mutate({
|
||||
source: "global",
|
||||
payload: { request_delay_sec: globalDelay },
|
||||
});
|
||||
}}
|
||||
>
|
||||
<label>
|
||||
Global delay (сек, 0–30)
|
||||
<input
|
||||
type="number"
|
||||
step={0.5}
|
||||
min={0}
|
||||
max={30}
|
||||
value={globalDelay}
|
||||
onChange={(e) => setGlobalDelay(parseFloat(e.target.value))}
|
||||
/>
|
||||
</label>
|
||||
<input
|
||||
type="range"
|
||||
min={0}
|
||||
max={30}
|
||||
step={0.5}
|
||||
value={globalDelay}
|
||||
onChange={(e) => setGlobalDelay(parseFloat(e.target.value))}
|
||||
style={{ width: "100%", gridColumn: "1 / -1" }}
|
||||
/>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={updateSettingMut.isPending}
|
||||
style={{ gridColumn: "1 / -1" }}
|
||||
>
|
||||
{updateSettingMut.isPending
|
||||
? "Сохраняем…"
|
||||
: "Сохранить global delay"}
|
||||
</button>
|
||||
</form>
|
||||
|
||||
{updateSettingMut.error && (
|
||||
<div className="scraper-result scraper-result--error">
|
||||
Ошибка: {updateSettingMut.error.message}
|
||||
</div>
|
||||
)}
|
||||
{updateSettingMut.isSuccess && updateSettingMut.data && (
|
||||
<div className="scraper-result">
|
||||
Delay обновлён: <strong>{updateSettingMut.data.request_delay_sec.toFixed(1)} сек</strong>{" "}
|
||||
(обновлено {formatTime(updateSettingMut.data.updated_at)})
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
</main>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||||
|
||||
function formatTime(iso: string | null): string {
|
||||
if (!iso) return "—";
|
||||
try {
|
||||
return new Date(iso).toLocaleString("ru-RU", {
|
||||
day: "2-digit",
|
||||
month: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
});
|
||||
} catch {
|
||||
return iso;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue