560 lines
18 KiB
TypeScript
560 lines
18 KiB
TypeScript
"use client";
|
||
|
||
import { useEffect, useState } from "react";
|
||
import Link from "next/link";
|
||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||
|
||
import "@/components/trade-in/trade-in.css";
|
||
import { Topbar } from "@/components/trade-in/Topbar";
|
||
import { apiFetch } from "@/lib/api";
|
||
|
||
// ── Types ──────────────────────────────────────────────────────────────────
|
||
|
||
interface ScrapeAroundResp {
|
||
total_fetched: number;
|
||
total_inserted: number;
|
||
total_updated: number;
|
||
by_source: Array<{
|
||
source: string;
|
||
fetched: number;
|
||
inserted: number;
|
||
updated: number;
|
||
}>;
|
||
}
|
||
|
||
interface TestAuthResp {
|
||
authenticated: boolean;
|
||
userId: number | null;
|
||
reason: string | null;
|
||
}
|
||
|
||
interface CianDetailTriggerResp {
|
||
ok: boolean;
|
||
offer_url: string;
|
||
cian_id: number | null;
|
||
price_changes_count: number;
|
||
saved: boolean;
|
||
listing_id: number | null;
|
||
}
|
||
|
||
interface CianNewbuildingTriggerResp {
|
||
ok: boolean;
|
||
zhk_url: string;
|
||
cian_internal_house_id: number | null;
|
||
name: string | null;
|
||
price_dynamics_count: number;
|
||
has_management_company: boolean;
|
||
saved: boolean;
|
||
house_id: number | null;
|
||
}
|
||
|
||
interface ScraperSetting {
|
||
source: string;
|
||
request_delay_sec: number;
|
||
description: string | null;
|
||
updated_at: string | null;
|
||
}
|
||
|
||
interface ScraperSettingUpdate {
|
||
request_delay_sec: number;
|
||
}
|
||
|
||
// ── Hooks ──────────────────────────────────────────────────────────────────
|
||
|
||
function useScrapeAround() {
|
||
return useMutation<
|
||
ScrapeAroundResp,
|
||
Error,
|
||
{ lat: number; lon: number; radius_m: number; multi_room_cian: boolean }
|
||
>({
|
||
mutationFn: (input) =>
|
||
apiFetch<ScrapeAroundResp>("/api/v1/admin/scrape", {
|
||
method: "POST",
|
||
body: JSON.stringify({ ...input, sources: ["cian"] }),
|
||
}),
|
||
});
|
||
}
|
||
|
||
function useTestAuth() {
|
||
return useQuery<TestAuthResp>({
|
||
queryKey: ["cian-test-auth"],
|
||
queryFn: () => apiFetch<TestAuthResp>("/api/v1/admin/scrape/cian/test-auth"),
|
||
staleTime: 30_000,
|
||
refetchInterval: 60_000,
|
||
});
|
||
}
|
||
|
||
function useScraperSettings() {
|
||
return useQuery<ScraperSetting[]>({
|
||
queryKey: ["scraper-settings"],
|
||
queryFn: () =>
|
||
apiFetch<ScraperSetting[]>("/api/v1/admin/scraper-settings"),
|
||
staleTime: 30_000,
|
||
});
|
||
}
|
||
|
||
function useUpdateScraperSetting() {
|
||
const qc = useQueryClient();
|
||
return useMutation<
|
||
ScraperSetting,
|
||
Error,
|
||
{ source: string; payload: ScraperSettingUpdate }
|
||
>({
|
||
mutationFn: ({ source, payload }) =>
|
||
apiFetch<ScraperSetting>(`/api/v1/admin/scraper-settings/${source}`, {
|
||
method: "PUT",
|
||
body: JSON.stringify(payload),
|
||
}),
|
||
onSuccess: () => {
|
||
qc.invalidateQueries({ queryKey: ["scraper-settings"] });
|
||
},
|
||
});
|
||
}
|
||
|
||
function useScrapeCianDetail() {
|
||
return useMutation<
|
||
CianDetailTriggerResp,
|
||
Error,
|
||
{ offer_url: string; listing_id: number | undefined }
|
||
>({
|
||
mutationFn: ({ offer_url, listing_id }) => {
|
||
const qs = new URLSearchParams({ offer_url });
|
||
if (listing_id !== undefined) qs.set("listing_id", String(listing_id));
|
||
return apiFetch<CianDetailTriggerResp>(
|
||
`/api/v1/admin/scrape/cian-detail?${qs.toString()}`,
|
||
{ method: "POST" },
|
||
);
|
||
},
|
||
});
|
||
}
|
||
|
||
function useScrapeCianNewbuilding() {
|
||
return useMutation<
|
||
CianNewbuildingTriggerResp,
|
||
Error,
|
||
{ zhk_url: string; house_id: number | undefined }
|
||
>({
|
||
mutationFn: ({ zhk_url, house_id }) => {
|
||
const qs = new URLSearchParams({ zhk_url });
|
||
if (house_id !== undefined) qs.set("house_id", String(house_id));
|
||
return apiFetch<CianNewbuildingTriggerResp>(
|
||
`/api/v1/admin/scrape/cian-newbuilding?${qs.toString()}`,
|
||
{ method: "POST" },
|
||
);
|
||
},
|
||
});
|
||
}
|
||
|
||
// ── Result panel ───────────────────────────────────────────────────────────
|
||
|
||
interface ResultPanelProps<TData> {
|
||
mut: {
|
||
data?: TData;
|
||
error: Error | null;
|
||
isPending: boolean;
|
||
isSuccess: boolean;
|
||
};
|
||
}
|
||
|
||
function ResultPanel<TData>({ mut }: ResultPanelProps<TData>) {
|
||
if (mut.error) {
|
||
return (
|
||
<div className="scraper-result scraper-result--error">
|
||
<strong>Ошибка:</strong> {mut.error.message}
|
||
</div>
|
||
);
|
||
}
|
||
if (mut.isSuccess && mut.data) {
|
||
return (
|
||
<div className="scraper-result">
|
||
<pre>{JSON.stringify(mut.data, null, 2)}</pre>
|
||
</div>
|
||
);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
// ── Page ───────────────────────────────────────────────────────────────────
|
||
|
||
export default function CianScraperPage() {
|
||
// Auth status
|
||
const authQ = useTestAuth();
|
||
|
||
// 1. Search around
|
||
const [lat, setLat] = useState("56.8400");
|
||
const [lon, setLon] = useState("60.6050");
|
||
const [radius, setRadius] = useState("1500");
|
||
const [multiRoom, setMultiRoom] = useState(false);
|
||
const aroundMut = useScrapeAround();
|
||
|
||
// 2. Detail by URL
|
||
const [detailUrl, setDetailUrl] = useState("");
|
||
const [detailListingId, setDetailListingId] = useState("");
|
||
const detailMut = useScrapeCianDetail();
|
||
|
||
// 3. Newbuilding
|
||
const [nbZhkUrl, setNbZhkUrl] = useState("");
|
||
const [nbHouseId, setNbHouseId] = useState("");
|
||
const nbMut = useScrapeCianNewbuilding();
|
||
|
||
// Global delay
|
||
const settingsQ = useScraperSettings();
|
||
const updateSettingMut = useUpdateScraperSetting();
|
||
|
||
const [globalDelay, setGlobalDelay] = useState<number>(5);
|
||
|
||
useEffect(() => {
|
||
const global = settingsQ.data?.find((s) => s.source === "global");
|
||
if (global !== undefined) {
|
||
setGlobalDelay(global.request_delay_sec);
|
||
}
|
||
}, [settingsQ.data]);
|
||
|
||
return (
|
||
<>
|
||
<Topbar active="cian" />
|
||
<main className="page scraper-page">
|
||
<h1 className="scraper-h1">Cian скрапер — admin триггер</h1>
|
||
<p className="scraper-subtitle">
|
||
Ручной запуск sub-pipeline'ов CianScraper. Для debug и теста после
|
||
изменений.
|
||
</p>
|
||
|
||
{/* Cookies callout */}
|
||
<div
|
||
className="scraper-section"
|
||
style={{
|
||
background: "var(--surface-2, #f0f4ff)",
|
||
borderLeft: "4px solid var(--accent, #3b6cff)",
|
||
padding: "12px 16px",
|
||
}}
|
||
>
|
||
<p style={{ margin: 0, fontSize: "0.9rem" }}>
|
||
<strong>Stage 7: Cian Valuation</strong> требует загруженных cookies.{" "}
|
||
<Link href="/scrapers/cian-cookies">
|
||
Настроить cookies →
|
||
</Link>
|
||
</p>
|
||
<p style={{ margin: "6px 0 0", fontSize: "0.85rem", color: "var(--muted, #5b6066)" }}>
|
||
Статус сессии:{" "}
|
||
{authQ.isPending && "проверяем…"}
|
||
{authQ.error && (
|
||
<span style={{ color: "var(--error, #c0392b)" }}>
|
||
ошибка проверки
|
||
</span>
|
||
)}
|
||
{authQ.data && (
|
||
<span
|
||
style={{
|
||
color: authQ.data.authenticated
|
||
? "var(--success, #27ae60)"
|
||
: "var(--error, #c0392b)",
|
||
fontWeight: 600,
|
||
}}
|
||
>
|
||
{authQ.data.authenticated
|
||
? `авторизован (userId: ${authQ.data.userId ?? "—"})`
|
||
: `не авторизован — ${authQ.data.reason ?? "неизвестная причина"}`}
|
||
</span>
|
||
)}
|
||
</p>
|
||
</div>
|
||
|
||
{/* 1. Search around point */}
|
||
<section className="scraper-section">
|
||
<h2>1. Search around (общий /admin/scrape)</h2>
|
||
<p className="scraper-hint">
|
||
Запускает Cian SERP pipeline для одного якоря: search → save_listings.
|
||
Поддерживает multi_room режим (~4× лотов, отдельный запрос на каждое
|
||
кол-во комнат).
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
aroundMut.mutate({
|
||
lat: parseFloat(lat),
|
||
lon: parseFloat(lon),
|
||
radius_m: parseInt(radius, 10),
|
||
multi_room_cian: multiRoom,
|
||
});
|
||
}}
|
||
>
|
||
<label>
|
||
Lat
|
||
<input
|
||
value={lat}
|
||
onChange={(e) => setLat(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Lon
|
||
<input
|
||
value={lon}
|
||
onChange={(e) => setLon(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Radius (м)
|
||
<input
|
||
value={radius}
|
||
onChange={(e) => setRadius(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label className="checkbox">
|
||
<input
|
||
type="checkbox"
|
||
checked={multiRoom}
|
||
onChange={(e) => setMultiRoom(e.target.checked)}
|
||
/>
|
||
multi_room_cian (4 запроса по комнатам)
|
||
</label>
|
||
<button type="submit" disabled={aroundMut.isPending}>
|
||
{aroundMut.isPending ? "Скрейпим…" : "Запустить"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={aroundMut} />
|
||
</section>
|
||
|
||
{/* 2. Detail by URL */}
|
||
<section className="scraper-section">
|
||
<h2>2. Detail page enrichment</h2>
|
||
<p className="scraper-hint">
|
||
Парсинг конкретного объявления Cian по URL: цена, история цен, сохранение
|
||
в cian_price_dynamics. Если <code>listing_id</code> не указан — только
|
||
debug-режим без записи в БД.
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
detailMut.mutate({
|
||
offer_url: detailUrl,
|
||
listing_id: detailListingId !== "" ? parseInt(detailListingId, 10) : undefined,
|
||
});
|
||
}}
|
||
>
|
||
<label className="full">
|
||
offer_url
|
||
<input
|
||
type="url"
|
||
value={detailUrl}
|
||
onChange={(e) => setDetailUrl(e.target.value)}
|
||
required
|
||
placeholder="https://ekb.cian.ru/sale/flat/XXXXX/"
|
||
/>
|
||
</label>
|
||
<label>
|
||
listing_id (опционально)
|
||
<input
|
||
type="number"
|
||
value={detailListingId}
|
||
onChange={(e) => setDetailListingId(e.target.value)}
|
||
placeholder="оставьте пустым для debug-режима"
|
||
/>
|
||
</label>
|
||
<button type="submit" disabled={detailMut.isPending}>
|
||
{detailMut.isPending ? "Парсим…" : "Запустить"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={detailMut} />
|
||
</section>
|
||
|
||
{/* 3. Newbuilding */}
|
||
<section className="scraper-section">
|
||
<h2>3. Новостройки (ЖК scrape)</h2>
|
||
<p className="scraper-hint">
|
||
Скрейп landing-страницы ЖК Cian: динамика цен, управляющая компания.
|
||
Если <code>house_id</code> не указан — только debug-режим без записи в БД.
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
nbMut.mutate({
|
||
zhk_url: nbZhkUrl,
|
||
house_id: nbHouseId !== "" ? parseInt(nbHouseId, 10) : undefined,
|
||
});
|
||
}}
|
||
>
|
||
<label className="full">
|
||
zhk_url
|
||
<input
|
||
type="url"
|
||
value={nbZhkUrl}
|
||
onChange={(e) => setNbZhkUrl(e.target.value)}
|
||
required
|
||
placeholder="https://zhk-XXX-ekb-i.cian.ru/"
|
||
/>
|
||
</label>
|
||
<label>
|
||
house_id (опционально)
|
||
<input
|
||
type="number"
|
||
value={nbHouseId}
|
||
onChange={(e) => setNbHouseId(e.target.value)}
|
||
placeholder="оставьте пустым для debug-режима"
|
||
/>
|
||
</label>
|
||
<button type="submit" disabled={nbMut.isPending}>
|
||
{nbMut.isPending ? "Парсим…" : "Запустить"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={nbMut} />
|
||
</section>
|
||
|
||
{/* 4. Valuation Calculator */}
|
||
<section className="scraper-section">
|
||
<h2>4. Valuation Calculator (оценка квартиры)</h2>
|
||
<p className="scraper-hint">
|
||
Calculator вызывается автоматически из /estimate flow (Stage 9).
|
||
Отдельный admin trigger будет добавлен позже.
|
||
</p>
|
||
</section>
|
||
|
||
{/* 5. Global Request Delay */}
|
||
<section className="scraper-section scraper-section--highlight">
|
||
<h2>5. Глобальный задержка запросов (Global Request Delay)</h2>
|
||
<p className="scraper-hint">
|
||
Глобальный делэй применяется ко всем scrapers (max с per-source
|
||
defaults). 0 = только per-source defaults. Изменение вступает в силу
|
||
немедленно — без перезапуска процесса.
|
||
</p>
|
||
|
||
{/* Per-source read-only list */}
|
||
{settingsQ.isPending && (
|
||
<p className="scraper-hint">Загрузка настроек…</p>
|
||
)}
|
||
{settingsQ.error && (
|
||
<p className="scraper-result scraper-result--error">
|
||
Ошибка загрузки: {settingsQ.error.message}
|
||
</p>
|
||
)}
|
||
{settingsQ.data && settingsQ.data.length > 0 && (
|
||
<div style={{ marginBottom: "12px" }}>
|
||
<p
|
||
style={{
|
||
fontSize: "0.85rem",
|
||
fontWeight: 600,
|
||
marginBottom: "4px",
|
||
}}
|
||
>
|
||
Текущие per-source задержки (read-only):
|
||
</p>
|
||
<table className="runs-table">
|
||
<thead>
|
||
<tr>
|
||
<th>Source</th>
|
||
<th>Delay (сек)</th>
|
||
<th>Описание</th>
|
||
<th>Обновлено</th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
{settingsQ.data.map((s) => (
|
||
<tr
|
||
key={s.source}
|
||
className={
|
||
s.source === "global" ? "run-row run-row--running" : "run-row"
|
||
}
|
||
>
|
||
<td>
|
||
<code>{s.source}</code>
|
||
</td>
|
||
<td>{s.request_delay_sec.toFixed(1)}</td>
|
||
<td
|
||
style={{
|
||
fontSize: "0.8rem",
|
||
color: "var(--muted, #5b6066)",
|
||
}}
|
||
>
|
||
{s.description ?? "—"}
|
||
</td>
|
||
<td style={{ fontSize: "0.8rem" }}>
|
||
{formatTime(s.updated_at)}
|
||
</td>
|
||
</tr>
|
||
))}
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
)}
|
||
|
||
{/* Global delay edit form */}
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
updateSettingMut.mutate({
|
||
source: "global",
|
||
payload: { request_delay_sec: globalDelay },
|
||
});
|
||
}}
|
||
>
|
||
<label>
|
||
Global delay (сек, 0–30)
|
||
<input
|
||
type="number"
|
||
step={0.5}
|
||
min={0}
|
||
max={30}
|
||
value={globalDelay}
|
||
onChange={(e) => setGlobalDelay(parseFloat(e.target.value))}
|
||
/>
|
||
</label>
|
||
<input
|
||
type="range"
|
||
min={0}
|
||
max={30}
|
||
step={0.5}
|
||
value={globalDelay}
|
||
onChange={(e) => setGlobalDelay(parseFloat(e.target.value))}
|
||
style={{ width: "100%", gridColumn: "1 / -1" }}
|
||
/>
|
||
<button
|
||
type="submit"
|
||
disabled={updateSettingMut.isPending}
|
||
style={{ gridColumn: "1 / -1" }}
|
||
>
|
||
{updateSettingMut.isPending
|
||
? "Сохраняем…"
|
||
: "Сохранить global delay"}
|
||
</button>
|
||
</form>
|
||
|
||
{updateSettingMut.error && (
|
||
<div className="scraper-result scraper-result--error">
|
||
Ошибка: {updateSettingMut.error.message}
|
||
</div>
|
||
)}
|
||
{updateSettingMut.isSuccess && updateSettingMut.data && (
|
||
<div className="scraper-result">
|
||
Delay обновлён: <strong>{updateSettingMut.data.request_delay_sec.toFixed(1)} сек</strong>{" "}
|
||
(обновлено {formatTime(updateSettingMut.data.updated_at)})
|
||
</div>
|
||
)}
|
||
</section>
|
||
</main>
|
||
</>
|
||
);
|
||
}
|
||
|
||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||
|
||
function formatTime(iso: string | null): string {
|
||
if (!iso) return "—";
|
||
try {
|
||
return new Date(iso).toLocaleString("ru-RU", {
|
||
day: "2-digit",
|
||
month: "2-digit",
|
||
hour: "2-digit",
|
||
minute: "2-digit",
|
||
second: "2-digit",
|
||
});
|
||
} catch {
|
||
return iso;
|
||
}
|
||
}
|