bot-frontend
1684c6ed3e
All checks were successful
Deploy Trade-In / changes (push) Successful in 5s
Deploy Trade-In / test (push) Has been skipped
Deploy Trade-In / build-backend (push) Has been skipped
Deploy Trade-In / build-frontend (push) Successful in 1m38s
Deploy Trade-In / deploy (push) Successful in 46s
Co-authored-by: bot-frontend <bot-frontend@gendsgn.local> Co-committed-by: bot-frontend <bot-frontend@gendsgn.local>
575 lines
19 KiB
TypeScript
575 lines
19 KiB
TypeScript
"use client";
|
||
|
||
import { useEffect, useRef, useState } from "react";
|
||
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
|
||
|
||
import "@/components/trade-in/trade-in.css";
|
||
import { Topbar } from "@/components/trade-in/Topbar";
|
||
import { apiFetch } from "@/lib/api";
|
||
|
||
// ── Types ──────────────────────────────────────────────────────────────────
|
||
|
||
interface ScraperSetting {
|
||
source: string;
|
||
request_delay_sec: number;
|
||
description: string | null;
|
||
updated_at: string | null;
|
||
}
|
||
|
||
interface ScraperSettingUpdate {
|
||
source: string;
|
||
request_delay_sec: number;
|
||
}
|
||
|
||
interface ScrapeAroundInput {
|
||
lat: number;
|
||
lon: number;
|
||
radius_m: number;
|
||
sources: string[];
|
||
multi_room_yandex?: boolean;
|
||
deep_yandex?: boolean;
|
||
}
|
||
|
||
interface ScrapeAroundResp {
|
||
total_fetched: number;
|
||
total_inserted: number;
|
||
total_updated: number;
|
||
by_source: Array<{
|
||
source: string;
|
||
fetched: number;
|
||
inserted: number;
|
||
updated: number;
|
||
}>;
|
||
}
|
||
|
||
interface YandexDetailTriggerResp {
|
||
ok: boolean;
|
||
offer_url: string;
|
||
offer_id: string | null;
|
||
price_rub: number | null;
|
||
title: string | null;
|
||
photo_count: number;
|
||
}
|
||
|
||
interface YandexNewbuildingTriggerResp {
|
||
ok: boolean;
|
||
ext_id: string;
|
||
ext_slug: string;
|
||
name: string | null;
|
||
lat: number | null;
|
||
lon: number | null;
|
||
rating: number | null;
|
||
ratings_count: number | null;
|
||
text_reviews_count: number | null;
|
||
developer_name: string | null;
|
||
}
|
||
|
||
interface YandexValuationTriggerResp {
|
||
ok: boolean;
|
||
address: string;
|
||
year_built: number | null;
|
||
total_floors: number | null;
|
||
house_type: string | null;
|
||
has_lift: boolean | null;
|
||
total_objects: number | null;
|
||
history_items_count: number;
|
||
}
|
||
|
||
// ── Hooks ──────────────────────────────────────────────────────────────────
|
||
|
||
function useScraperSettings() {
|
||
return useQuery<ScraperSetting[]>({
|
||
queryKey: ["scraper-settings"],
|
||
queryFn: () =>
|
||
apiFetch<{ settings: ScraperSetting[] }>("/api/v1/admin/scraper-settings").then(
|
||
(r) =>
|
||
(r.settings ?? []).map((s) => ({
|
||
...s,
|
||
request_delay_sec: Number(s.request_delay_sec),
|
||
})),
|
||
),
|
||
refetchInterval: 30_000,
|
||
});
|
||
}
|
||
|
||
function useUpdateScraperSetting() {
|
||
const qc = useQueryClient();
|
||
return useMutation<
|
||
ScraperSetting,
|
||
Error,
|
||
{ source: string; payload: Omit<ScraperSettingUpdate, "source"> }
|
||
>({
|
||
mutationFn: ({ source, payload }) =>
|
||
apiFetch<ScraperSetting>(`/api/v1/admin/scraper-settings/${source}`, {
|
||
method: "PUT",
|
||
body: JSON.stringify({ source, ...payload }),
|
||
}),
|
||
onSuccess: () => {
|
||
qc.invalidateQueries({ queryKey: ["scraper-settings"] });
|
||
},
|
||
});
|
||
}
|
||
|
||
function useScrapeAround() {
|
||
return useMutation<ScrapeAroundResp, Error, ScrapeAroundInput>({
|
||
mutationFn: (input) =>
|
||
apiFetch<ScrapeAroundResp>("/api/v1/admin/scrape", {
|
||
method: "POST",
|
||
body: JSON.stringify(input),
|
||
}),
|
||
});
|
||
}
|
||
|
||
function useScrapeYandexDetail() {
|
||
return useMutation<YandexDetailTriggerResp, Error, { offer_url: string }>({
|
||
mutationFn: ({ offer_url }) =>
|
||
apiFetch<YandexDetailTriggerResp>(
|
||
`/api/v1/admin/scrape/yandex-detail?offer_url=${encodeURIComponent(offer_url)}`,
|
||
{ method: "POST" },
|
||
),
|
||
});
|
||
}
|
||
|
||
function useScrapeYandexNewbuilding() {
|
||
return useMutation<
|
||
YandexNewbuildingTriggerResp,
|
||
Error,
|
||
{ slug: string; id: string; city: string }
|
||
>({
|
||
mutationFn: ({ slug, id, city }) => {
|
||
const qs = new URLSearchParams({ slug, id, city });
|
||
return apiFetch<YandexNewbuildingTriggerResp>(
|
||
`/api/v1/admin/scrape/yandex-newbuilding?${qs.toString()}`,
|
||
{ method: "POST" },
|
||
);
|
||
},
|
||
});
|
||
}
|
||
|
||
function useScrapeYandexValuation() {
|
||
return useMutation<
|
||
YandexValuationTriggerResp,
|
||
Error,
|
||
{ address: string; offer_category: string; offer_type: string; page: number }
|
||
>({
|
||
mutationFn: ({ address, offer_category, offer_type, page }) => {
|
||
const qs = new URLSearchParams({
|
||
address,
|
||
offer_category,
|
||
offer_type,
|
||
page: String(page),
|
||
});
|
||
return apiFetch<YandexValuationTriggerResp>(
|
||
`/api/v1/admin/scrape/yandex-valuation?${qs.toString()}`,
|
||
{ method: "POST" },
|
||
);
|
||
},
|
||
});
|
||
}
|
||
|
||
// ── Result panel ───────────────────────────────────────────────────────────
|
||
|
||
interface ResultPanelProps<TData> {
|
||
mut: {
|
||
data?: TData;
|
||
error: Error | null;
|
||
isPending: boolean;
|
||
isSuccess: boolean;
|
||
};
|
||
}
|
||
|
||
function ResultPanel<TData>({ mut }: ResultPanelProps<TData>) {
|
||
if (mut.error) {
|
||
return (
|
||
<div className="scraper-result scraper-result--error">
|
||
<strong>Ошибка:</strong> {mut.error.message}
|
||
</div>
|
||
);
|
||
}
|
||
if (mut.isSuccess && mut.data) {
|
||
return (
|
||
<div className="scraper-result">
|
||
<pre>{JSON.stringify(mut.data, null, 2)}</pre>
|
||
</div>
|
||
);
|
||
}
|
||
return null;
|
||
}
|
||
|
||
// ── Helpers ────────────────────────────────────────────────────────────────
|
||
|
||
function formatTime(iso: string | null): string {
|
||
if (!iso) return "—";
|
||
try {
|
||
return new Date(iso).toLocaleString("ru-RU", {
|
||
day: "2-digit",
|
||
month: "2-digit",
|
||
hour: "2-digit",
|
||
minute: "2-digit",
|
||
second: "2-digit",
|
||
});
|
||
} catch {
|
||
return iso;
|
||
}
|
||
}
|
||
|
||
// ── Page ───────────────────────────────────────────────────────────────────
|
||
|
||
export default function YandexScraperPage() {
|
||
// Settings
|
||
const settingsQ = useScraperSettings();
|
||
const updateSettingMut = useUpdateScraperSetting();
|
||
const [delayInput, setDelayInput] = useState<string>("5.0");
|
||
const delayInitializedRef = useRef(false);
|
||
|
||
useEffect(() => {
|
||
const yandex = settingsQ.data?.find((s) => s.source === "yandex");
|
||
if (yandex && !delayInitializedRef.current) {
|
||
setDelayInput(String(yandex.request_delay_sec));
|
||
delayInitializedRef.current = true;
|
||
}
|
||
}, [settingsQ.data]);
|
||
|
||
// Around
|
||
const [lat, setLat] = useState("56.8400");
|
||
const [lon, setLon] = useState("60.6050");
|
||
const [radius, setRadius] = useState("1500");
|
||
const [multiRoom, setMultiRoom] = useState(false);
|
||
const [deep, setDeep] = useState(false);
|
||
const aroundMut = useScrapeAround();
|
||
|
||
// Detail
|
||
const [detailUrl, setDetailUrl] = useState(
|
||
"https://realty.yandex.ru/offer/7567094292504417257/",
|
||
);
|
||
const detailMut = useScrapeYandexDetail();
|
||
|
||
// Newbuilding
|
||
const [nbSlug, setNbSlug] = useState("tatlin");
|
||
const [nbId, setNbId] = useState("1592987");
|
||
const [nbCity, setNbCity] = useState("ekaterinburg");
|
||
const nbMut = useScrapeYandexNewbuilding();
|
||
|
||
// Valuation
|
||
const [valAddress, setValAddress] = useState(
|
||
"Свердловская область, Екатеринбург, улица Учителей, 18",
|
||
);
|
||
const [valCategory, setValCategory] = useState("APARTMENT");
|
||
const [valType, setValType] = useState("SELL");
|
||
const [valPage, setValPage] = useState("1");
|
||
const valMut = useScrapeYandexValuation();
|
||
|
||
const yandexSetting = settingsQ.data?.find((s) => s.source === "yandex");
|
||
|
||
return (
|
||
<>
|
||
<Topbar active="yandex" />
|
||
<main className="page scraper-page">
|
||
<h1 className="scraper-h1">Yandex скрапер — admin триггер</h1>
|
||
<p className="scraper-subtitle">
|
||
Ручной запуск sub-pipeline'ов YandexRealtyScraper v1 + глобальная настройка
|
||
задержки между запросами.
|
||
</p>
|
||
|
||
{/* 0. Global delay */}
|
||
<section className="scraper-section scraper-section--highlight">
|
||
<h2>Глобальная задержка запросов</h2>
|
||
<p className="scraper-hint">
|
||
Настройка применяется ко ВСЕМ Yandex-скрейперам (SERP, detail, ЖК, valuation)
|
||
при создании следующего инстанса (in-process cache 60s). Меняется без рестарта
|
||
backend.
|
||
</p>
|
||
|
||
{settingsQ.isPending && <p className="scraper-hint">Загрузка настроек…</p>}
|
||
{settingsQ.error && (
|
||
<p className="scraper-result scraper-result--error">
|
||
Ошибка загрузки настроек: {settingsQ.error.message}
|
||
</p>
|
||
)}
|
||
|
||
{yandexSetting && (
|
||
<div className="schedule-status" style={{ marginBottom: "12px" }}>
|
||
<div className="schedule-status__row">
|
||
<span className="schedule-status__label">Текущая задержка:</span>
|
||
<span>
|
||
<strong>{yandexSetting.request_delay_sec} сек</strong>
|
||
</span>
|
||
</div>
|
||
{yandexSetting.updated_at && (
|
||
<div className="schedule-status__row">
|
||
<span className="schedule-status__label">Обновлено:</span>
|
||
<span>{formatTime(yandexSetting.updated_at)}</span>
|
||
</div>
|
||
)}
|
||
{yandexSetting.description && (
|
||
<div className="schedule-status__row">
|
||
<span className="schedule-status__label">Описание:</span>
|
||
<span className="run-muted">{yandexSetting.description}</span>
|
||
</div>
|
||
)}
|
||
</div>
|
||
)}
|
||
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
updateSettingMut.mutate({
|
||
source: "yandex",
|
||
payload: { request_delay_sec: parseFloat(delayInput) },
|
||
});
|
||
}}
|
||
>
|
||
<label>
|
||
Задержка (сек, 1–60)
|
||
<input
|
||
type="number"
|
||
min={1}
|
||
max={60}
|
||
step={0.5}
|
||
value={delayInput}
|
||
onChange={(e) => setDelayInput(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<button type="submit" disabled={updateSettingMut.isPending}>
|
||
{updateSettingMut.isPending ? "Сохраняем…" : "Сохранить"}
|
||
</button>
|
||
</form>
|
||
|
||
{updateSettingMut.error && (
|
||
<div className="scraper-result scraper-result--error">
|
||
<strong>Ошибка:</strong> {updateSettingMut.error.message}
|
||
</div>
|
||
)}
|
||
{updateSettingMut.isSuccess && updateSettingMut.data && (
|
||
<div className="scraper-result">
|
||
Сохранено. Новая задержка:{" "}
|
||
<strong>{updateSettingMut.data.request_delay_sec} сек</strong>
|
||
</div>
|
||
)}
|
||
</section>
|
||
|
||
{/* 1. Around point */}
|
||
<section className="scraper-section">
|
||
<h2>1. Search around (общий /admin/scrape)</h2>
|
||
<p className="scraper-hint">
|
||
Yandex SERP path-based vtorichka — lat/lon/radius логируется, scraper берёт{" "}
|
||
<code>/{ekaterinburg}/kupit/kvartira/vtorichniy-rynok/</code>.
|
||
multi-room split не реализован для DOM-парсера (запросы будут одинаковыми).
|
||
Deep пробует pages × sorts комбинации.
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
aroundMut.mutate({
|
||
lat: parseFloat(lat),
|
||
lon: parseFloat(lon),
|
||
radius_m: parseInt(radius, 10),
|
||
sources: ["yandex"],
|
||
multi_room_yandex: multiRoom,
|
||
deep_yandex: deep,
|
||
});
|
||
}}
|
||
>
|
||
<label>
|
||
Lat
|
||
<input
|
||
value={lat}
|
||
onChange={(e) => setLat(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Lon
|
||
<input
|
||
value={lon}
|
||
onChange={(e) => setLon(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Radius (м)
|
||
<input
|
||
type="number"
|
||
min={100}
|
||
max={10000}
|
||
step={100}
|
||
value={radius}
|
||
onChange={(e) => setRadius(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label className="checkbox">
|
||
<input
|
||
type="checkbox"
|
||
checked={multiRoom}
|
||
onChange={(e) => setMultiRoom(e.target.checked)}
|
||
/>
|
||
multi_room_yandex (5 rooms split)
|
||
</label>
|
||
<label className="checkbox">
|
||
<input
|
||
type="checkbox"
|
||
checked={deep}
|
||
onChange={(e) => setDeep(e.target.checked)}
|
||
/>
|
||
deep_yandex (5 × 3 × 2 = 30 requests)
|
||
</label>
|
||
<button type="submit" disabled={aroundMut.isPending}>
|
||
{aroundMut.isPending ? "Скрейпим…" : "Запустить"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={aroundMut} />
|
||
</section>
|
||
|
||
{/* 2. Detail */}
|
||
<section className="scraper-section">
|
||
<h2>2. Detail page enrichment</h2>
|
||
<p className="scraper-hint">
|
||
Product JSON-LD price (exact int) + DOM-секции description/agency/metro/photos.
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
detailMut.mutate({ offer_url: detailUrl });
|
||
}}
|
||
>
|
||
<label className="full">
|
||
offer_url
|
||
<input
|
||
value={detailUrl}
|
||
onChange={(e) => setDetailUrl(e.target.value)}
|
||
required
|
||
placeholder="https://realty.yandex.ru/offer/..."
|
||
/>
|
||
</label>
|
||
<button type="submit" disabled={detailMut.isPending}>
|
||
{detailMut.isPending ? "Парсим…" : "Запустить"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={detailMut} />
|
||
</section>
|
||
|
||
{/* 3. ЖК Newbuilding */}
|
||
<section className="scraper-section">
|
||
<h2>3. ЖК (Newbuilding) landing</h2>
|
||
<p className="scraper-hint">
|
||
URL /{city}/kupit/novostrojka/{slug}-{id}/ — извлекает coords inline,
|
||
rating, text_reviews_count, developer_name.
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
nbMut.mutate({ slug: nbSlug, id: nbId, city: nbCity });
|
||
}}
|
||
>
|
||
<label>
|
||
slug
|
||
<input
|
||
value={nbSlug}
|
||
onChange={(e) => setNbSlug(e.target.value)}
|
||
required
|
||
placeholder="tatlin"
|
||
/>
|
||
</label>
|
||
<label>
|
||
id
|
||
<input
|
||
value={nbId}
|
||
onChange={(e) => setNbId(e.target.value)}
|
||
required
|
||
placeholder="1592987"
|
||
/>
|
||
</label>
|
||
<label>
|
||
city
|
||
<input
|
||
value={nbCity}
|
||
onChange={(e) => setNbCity(e.target.value)}
|
||
required
|
||
placeholder="ekaterinburg"
|
||
/>
|
||
</label>
|
||
<button type="submit" disabled={nbMut.isPending}>
|
||
{nbMut.isPending ? "Парсим…" : "Запустить"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={nbMut} />
|
||
</section>
|
||
|
||
{/* 4. Valuation */}
|
||
<section className="scraper-section">
|
||
<h2>4. Valuation (anonymous history)</h2>
|
||
<p className="scraper-hint">
|
||
Anonymous GET /otsenka-kvartiry-po-adresu-onlayn/ — house meta + до 30 history
|
||
items на page.
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
valMut.mutate({
|
||
address: valAddress,
|
||
offer_category: valCategory,
|
||
offer_type: valType,
|
||
page: parseInt(valPage, 10),
|
||
});
|
||
}}
|
||
>
|
||
<label className="full">
|
||
Адрес
|
||
<input
|
||
value={valAddress}
|
||
onChange={(e) => setValAddress(e.target.value)}
|
||
required
|
||
placeholder="Свердловская область, Екатеринбург, улица Учителей, 18"
|
||
/>
|
||
</label>
|
||
<label>
|
||
offer_category
|
||
<select
|
||
value={valCategory}
|
||
onChange={(e) => setValCategory(e.target.value)}
|
||
>
|
||
<option value="APARTMENT">APARTMENT</option>
|
||
<option value="HOUSE">HOUSE</option>
|
||
<option value="GARAGE">GARAGE</option>
|
||
<option value="COMMERCIAL">COMMERCIAL</option>
|
||
<option value="ROOM">ROOM</option>
|
||
</select>
|
||
</label>
|
||
<label>
|
||
offer_type
|
||
<select
|
||
value={valType}
|
||
onChange={(e) => setValType(e.target.value)}
|
||
>
|
||
<option value="SELL">SELL</option>
|
||
<option value="RENT">RENT</option>
|
||
</select>
|
||
</label>
|
||
<label>
|
||
page
|
||
<input
|
||
type="number"
|
||
min={1}
|
||
max={100}
|
||
value={valPage}
|
||
onChange={(e) => setValPage(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<button type="submit" disabled={valMut.isPending}>
|
||
{valMut.isPending ? "Запрашиваем…" : "Запустить"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={valMut} />
|
||
</section>
|
||
</main>
|
||
</>
|
||
);
|
||
}
|