"use client"; import { useEffect, useRef, useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import "@/components/trade-in/trade-in.css"; import { Topbar } from "@/components/trade-in/Topbar"; import { apiFetch } from "@/lib/api"; // ── Types ────────────────────────────────────────────────────────────────── interface ScraperSetting { source: string; request_delay_sec: number; description: string | null; updated_at: string | null; } interface ScraperSettingUpdate { source: string; request_delay_sec: number; } interface ScrapeAroundInput { lat: number; lon: number; radius_m: number; sources: string[]; multi_room_yandex?: boolean; deep_yandex?: boolean; } interface ScrapeAroundResp { total_fetched: number; total_inserted: number; total_updated: number; by_source: Array<{ source: string; fetched: number; inserted: number; updated: number; }>; } interface YandexDetailTriggerResp { ok: boolean; offer_url: string; offer_id: string | null; price_rub: number | null; title: string | null; photo_count: number; } interface YandexNewbuildingTriggerResp { ok: boolean; ext_id: string; ext_slug: string; name: string | null; lat: number | null; lon: number | null; rating: number | null; ratings_count: number | null; text_reviews_count: number | null; developer_name: string | null; } interface YandexValuationTriggerResp { ok: boolean; address: string; year_built: number | null; total_floors: number | null; house_type: string | null; has_lift: boolean | null; total_objects: number | null; history_items_count: number; } // ── Hooks ────────────────────────────────────────────────────────────────── function useScraperSettings() { return useQuery({ queryKey: ["scraper-settings"], queryFn: () => apiFetch<{ settings: ScraperSetting[] }>("/api/v1/admin/scraper-settings").then( (r) => (r.settings ?? []).map((s) => ({ ...s, request_delay_sec: Number(s.request_delay_sec), })), ), refetchInterval: 30_000, }); } function useUpdateScraperSetting() { const qc = useQueryClient(); return useMutation< ScraperSetting, Error, { source: string; payload: Omit } >({ mutationFn: ({ source, payload }) => apiFetch(`/api/v1/admin/scraper-settings/${source}`, { method: "PUT", body: JSON.stringify({ source, ...payload }), }), onSuccess: () => { qc.invalidateQueries({ queryKey: ["scraper-settings"] }); }, }); } function useScrapeAround() { return useMutation({ mutationFn: (input) => apiFetch("/api/v1/admin/scrape", { method: "POST", body: JSON.stringify(input), }), }); } function useScrapeYandexDetail() { return useMutation({ mutationFn: ({ offer_url }) => apiFetch( `/api/v1/admin/scrape/yandex-detail?offer_url=${encodeURIComponent(offer_url)}`, { method: "POST" }, ), }); } function useScrapeYandexNewbuilding() { return useMutation< YandexNewbuildingTriggerResp, Error, { slug: string; id: string; city: string } >({ mutationFn: ({ slug, id, city }) => { const qs = new URLSearchParams({ slug, id, city }); return apiFetch( `/api/v1/admin/scrape/yandex-newbuilding?${qs.toString()}`, { method: "POST" }, ); }, }); } function useScrapeYandexValuation() { return useMutation< YandexValuationTriggerResp, Error, { address: string; offer_category: string; offer_type: string; page: number } >({ mutationFn: ({ address, offer_category, offer_type, page }) => { const qs = new URLSearchParams({ address, offer_category, offer_type, page: String(page), }); return apiFetch( `/api/v1/admin/scrape/yandex-valuation?${qs.toString()}`, { method: "POST" }, ); }, }); } // ── Result panel ─────────────────────────────────────────────────────────── interface ResultPanelProps { mut: { data?: TData; error: Error | null; isPending: boolean; isSuccess: boolean; }; } function ResultPanel({ mut }: ResultPanelProps) { if (mut.error) { return (
Ошибка: {mut.error.message}
); } if (mut.isSuccess && mut.data) { return (
{JSON.stringify(mut.data, null, 2)}
); } return null; } // ── Helpers ──────────────────────────────────────────────────────────────── function formatTime(iso: string | null): string { if (!iso) return "—"; try { return new Date(iso).toLocaleString("ru-RU", { day: "2-digit", month: "2-digit", hour: "2-digit", minute: "2-digit", second: "2-digit", }); } catch { return iso; } } // ── Page ─────────────────────────────────────────────────────────────────── export default function YandexScraperPage() { // Settings const settingsQ = useScraperSettings(); const updateSettingMut = useUpdateScraperSetting(); const [delayInput, setDelayInput] = useState("5.0"); const delayInitializedRef = useRef(false); useEffect(() => { const yandex = settingsQ.data?.find((s) => s.source === "yandex"); if (yandex && !delayInitializedRef.current) { setDelayInput(String(yandex.request_delay_sec)); delayInitializedRef.current = true; } }, [settingsQ.data]); // Around const [lat, setLat] = useState("56.8400"); const [lon, setLon] = useState("60.6050"); const [radius, setRadius] = useState("1500"); const [multiRoom, setMultiRoom] = useState(false); const [deep, setDeep] = useState(false); const aroundMut = useScrapeAround(); // Detail const [detailUrl, setDetailUrl] = useState( "https://realty.yandex.ru/offer/7567094292504417257/", ); const detailMut = useScrapeYandexDetail(); // Newbuilding const [nbSlug, setNbSlug] = useState("tatlin"); const [nbId, setNbId] = useState("1592987"); const [nbCity, setNbCity] = useState("ekaterinburg"); const nbMut = useScrapeYandexNewbuilding(); // Valuation const [valAddress, setValAddress] = useState( "Свердловская область, Екатеринбург, улица Учителей, 18", ); const [valCategory, setValCategory] = useState("APARTMENT"); const [valType, setValType] = useState("SELL"); const [valPage, setValPage] = useState("1"); const valMut = useScrapeYandexValuation(); const yandexSetting = settingsQ.data?.find((s) => s.source === "yandex"); return ( <>

Yandex скрапер — admin триггер

Ручной запуск sub-pipeline'ов YandexRealtyScraper v1 + глобальная настройка задержки между запросами.

{/* 0. Global delay */}

Глобальная задержка запросов

Настройка применяется ко ВСЕМ Yandex-скрейперам (SERP, detail, ЖК, valuation) при создании следующего инстанса (in-process cache 60s). Меняется без рестарта backend.

{settingsQ.isPending &&

Загрузка настроек…

} {settingsQ.error && (

Ошибка загрузки настроек: {settingsQ.error.message}

)} {yandexSetting && (
Текущая задержка: {yandexSetting.request_delay_sec} сек
{yandexSetting.updated_at && (
Обновлено: {formatTime(yandexSetting.updated_at)}
)} {yandexSetting.description && (
Описание: {yandexSetting.description}
)}
)}
{ e.preventDefault(); updateSettingMut.mutate({ source: "yandex", payload: { request_delay_sec: parseFloat(delayInput) }, }); }} >
{updateSettingMut.error && (
Ошибка: {updateSettingMut.error.message}
)} {updateSettingMut.isSuccess && updateSettingMut.data && (
Сохранено. Новая задержка:{" "} {updateSettingMut.data.request_delay_sec} сек
)}
{/* 1. Around point */}

1. Search around (общий /admin/scrape)

Yandex SERP path-based vtorichka — lat/lon/radius логируется, scraper берёт{" "} /{ekaterinburg}/kupit/kvartira/vtorichniy-rynok/. multi-room split не реализован для DOM-парсера (запросы будут одинаковыми). Deep пробует pages × sorts комбинации.

{ e.preventDefault(); aroundMut.mutate({ lat: parseFloat(lat), lon: parseFloat(lon), radius_m: parseInt(radius, 10), sources: ["yandex"], multi_room_yandex: multiRoom, deep_yandex: deep, }); }} >
{/* 2. Detail */}

2. Detail page enrichment

Product JSON-LD price (exact int) + DOM-секции description/agency/metro/photos.

{ e.preventDefault(); detailMut.mutate({ offer_url: detailUrl }); }} >
{/* 3. ЖК Newbuilding */}

3. ЖК (Newbuilding) landing

URL /{city}/kupit/novostrojka/{slug}-{id}/ — извлекает coords inline, rating, text_reviews_count, developer_name.

{ e.preventDefault(); nbMut.mutate({ slug: nbSlug, id: nbId, city: nbCity }); }} >
{/* 4. Valuation */}

4. Valuation (anonymous history)

Anonymous GET /otsenka-kvartiry-po-adresu-onlayn/ — house meta + до 30 history items на page.

{ e.preventDefault(); valMut.mutate({ address: valAddress, offer_category: valCategory, offer_type: valType, page: parseInt(valPage, 10), }); }} >
); }