diff --git a/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx b/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx index 47506196..ef9059c9 100644 --- a/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx +++ b/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx @@ -1,23 +1,570 @@ "use client"; +import { useEffect, useRef, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + import "@/components/trade-in/trade-in.css"; import { Topbar } from "@/components/trade-in/Topbar"; +import { apiFetch } from "@/lib/api"; + +// ── Types ────────────────────────────────────────────────────────────────── + +interface ScraperSetting { + source: string; + request_delay_sec: number; + description: string | null; + updated_at: string | null; +} + +interface ScraperSettingUpdate { + source: string; + request_delay_sec: number; +} + +interface ScrapeAroundInput { + lat: number; + lon: number; + radius_m: number; + sources: string[]; + multi_room_yandex?: boolean; + deep_yandex?: boolean; +} + +interface ScrapeAroundResp { + total_fetched: number; + total_inserted: number; + total_updated: number; + by_source: Array<{ + source: string; + fetched: number; + inserted: number; + updated: number; + }>; +} + +interface YandexDetailTriggerResp { + ok: boolean; + offer_url: string; + offer_id: string | null; + price_rub: number | null; + title: string | null; + photo_count: number; +} + +interface YandexNewbuildingTriggerResp { + ok: boolean; + ext_id: string; + ext_slug: string; + name: string | null; + lat: number | null; + lon: number | null; + rating: number | null; + ratings_count: number | null; + text_reviews_count: number | null; + developer_name: string | null; +} + +interface YandexValuationTriggerResp { + ok: boolean; + address: string; + year_built: number | null; + total_floors: number | null; + house_type: string | null; + has_lift: boolean | null; + total_objects: number | null; + history_items_count: number; +} + +// ── Hooks ────────────────────────────────────────────────────────────────── + +function useScraperSettings() { + return useQuery({ + queryKey: ["scraper-settings"], + queryFn: () => + apiFetch<{ settings: ScraperSetting[] }>("/api/v1/admin/scraper-settings").then( + (r) => r.settings, + ), + refetchInterval: 30_000, + }); +} + +function useUpdateScraperSetting() { + const qc = useQueryClient(); + return useMutation< + ScraperSetting, + Error, + { source: string; payload: Omit } + >({ + mutationFn: ({ source, payload }) => + apiFetch(`/api/v1/admin/scraper-settings/${source}`, { + method: "PUT", + body: JSON.stringify({ source, ...payload }), + }), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["scraper-settings"] }); + }, + }); +} + +function useScrapeAround() { + return useMutation({ + mutationFn: (input) => + apiFetch("/api/v1/admin/scrape", { + method: "POST", + body: JSON.stringify(input), + }), + }); +} + +function useScrapeYandexDetail() { + return useMutation({ + mutationFn: ({ offer_url }) => + apiFetch( + `/api/v1/admin/scrape/yandex-detail?offer_url=${encodeURIComponent(offer_url)}`, + { method: "POST" }, + ), + }); +} + +function useScrapeYandexNewbuilding() { + return useMutation< + YandexNewbuildingTriggerResp, + Error, + { slug: string; id: string; city: string } + >({ + mutationFn: ({ slug, id, city }) => { + const qs = new URLSearchParams({ slug, id, city }); + return apiFetch( + `/api/v1/admin/scrape/yandex-newbuilding?${qs.toString()}`, + { method: "POST" }, + ); + }, + }); +} + +function useScrapeYandexValuation() { + return useMutation< + YandexValuationTriggerResp, + Error, + { address: string; offer_category: string; offer_type: string; page: number } + >({ + mutationFn: ({ address, offer_category, offer_type, page }) => { + const qs = new URLSearchParams({ + address, + offer_category, + offer_type, + page: String(page), + }); + return apiFetch( + `/api/v1/admin/scrape/yandex-valuation?${qs.toString()}`, + { method: "POST" }, + ); + }, + }); +} + +// ── Result panel ─────────────────────────────────────────────────────────── + +interface ResultPanelProps { + mut: { + data?: TData; + error: Error | null; + isPending: boolean; + isSuccess: boolean; + }; +} + +function ResultPanel({ mut }: ResultPanelProps) { + if (mut.error) { + return ( +
+ Ошибка: {mut.error.message} +
+ ); + } + if (mut.isSuccess && mut.data) { + return ( +
+
{JSON.stringify(mut.data, null, 2)}
+
+ ); + } + return null; +} + +// ── Helpers ──────────────────────────────────────────────────────────────── + +function formatTime(iso: string | null): string { + if (!iso) return "—"; + try { + return new Date(iso).toLocaleString("ru-RU", { + day: "2-digit", + month: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); + } catch { + return iso; + } +} + +// ── Page ─────────────────────────────────────────────────────────────────── export default function YandexScraperPage() { + // Settings + const settingsQ = useScraperSettings(); + const updateSettingMut = useUpdateScraperSetting(); + const [delayInput, setDelayInput] = useState("5.0"); + const delayInitializedRef = useRef(false); + + useEffect(() => { + const yandex = settingsQ.data?.find((s) => s.source === "yandex"); + if (yandex && !delayInitializedRef.current) { + setDelayInput(String(yandex.request_delay_sec)); + delayInitializedRef.current = true; + } + }, [settingsQ.data]); + + // Around + const [lat, setLat] = useState("56.8400"); + const [lon, setLon] = useState("60.6050"); + const [radius, setRadius] = useState("1500"); + const [multiRoom, setMultiRoom] = useState(false); + const [deep, setDeep] = useState(false); + const aroundMut = useScrapeAround(); + + // Detail + const [detailUrl, setDetailUrl] = useState( + "https://realty.yandex.ru/offer/7567094292504417257/", + ); + const detailMut = useScrapeYandexDetail(); + + // Newbuilding + const [nbSlug, setNbSlug] = useState("tatlin"); + const [nbId, setNbId] = useState("1592987"); + const [nbCity, setNbCity] = useState("ekaterinburg"); + const nbMut = useScrapeYandexNewbuilding(); + + // Valuation + const [valAddress, setValAddress] = useState( + "Свердловская область, Екатеринбург, улица Учителей, 18", + ); + const [valCategory, setValCategory] = useState("APARTMENT"); + const [valType, setValType] = useState("SELL"); + const [valPage, setValPage] = useState("1"); + const valMut = useScrapeYandexValuation(); + + const yandexSetting = settingsQ.data?.find((s) => s.source === "yandex"); + return ( <>
-

Yandex скрапер

-

В разработке.

-
-

- Yandex SERP refactor merged (PR #462), но admin UI ещё не подключён. - Используй cron-scrape.sh или прямой curl к{" "} - POST /api/v1/admin/scrape с{" "} - sources: ["yandex"]. +

Yandex скрапер — admin триггер

+

+ Ручной запуск sub-pipeline'ов YandexRealtyScraper v1 + глобальная настройка + задержки между запросами. +

+ + {/* 0. Global delay */} +
+

Глобальная задержка запросов

+

+ Настройка применяется ко ВСЕМ Yandex-скрейперам (SERP, detail, ЖК, valuation) + при создании следующего инстанса (in-process cache 60s). Меняется без рестарта + backend.

-
+ + {settingsQ.isPending &&

Загрузка настроек…

} + {settingsQ.error && ( +

+ Ошибка загрузки настроек: {settingsQ.error.message} +

+ )} + + {yandexSetting && ( +
+
+ Текущая задержка: + + {yandexSetting.request_delay_sec} сек + +
+ {yandexSetting.updated_at && ( +
+ Обновлено: + {formatTime(yandexSetting.updated_at)} +
+ )} + {yandexSetting.description && ( +
+ Описание: + {yandexSetting.description} +
+ )} +
+ )} + +
{ + e.preventDefault(); + updateSettingMut.mutate({ + source: "yandex", + payload: { request_delay_sec: parseFloat(delayInput) }, + }); + }} + > + + +
+ + {updateSettingMut.error && ( +
+ Ошибка: {updateSettingMut.error.message} +
+ )} + {updateSettingMut.isSuccess && updateSettingMut.data && ( +
+ Сохранено. Новая задержка:{" "} + {updateSettingMut.data.request_delay_sec} сек +
+ )} + + + {/* 1. Around point */} +
+

1. Search around (общий /admin/scrape)

+

+ Yandex SERP path-based vtorichka — lat/lon/radius логируется, scraper берёт{" "} + /{ekaterinburg}/kupit/kvartira/vtorichniy-rynok/. + multi-room split не реализован для DOM-парсера (запросы будут одинаковыми). + Deep пробует pages × sorts комбинации. +

+
{ + e.preventDefault(); + aroundMut.mutate({ + lat: parseFloat(lat), + lon: parseFloat(lon), + radius_m: parseInt(radius, 10), + sources: ["yandex"], + multi_room_yandex: multiRoom, + deep_yandex: deep, + }); + }} + > + + + + + + +
+ +
+ + {/* 2. Detail */} +
+

2. Detail page enrichment

+

+ Product JSON-LD price (exact int) + DOM-секции description/agency/metro/photos. +

+
{ + e.preventDefault(); + detailMut.mutate({ offer_url: detailUrl }); + }} + > + + +
+ +
+ + {/* 3. ЖК Newbuilding */} +
+

3. ЖК (Newbuilding) landing

+

+ URL /{city}/kupit/novostrojka/{slug}-{id}/ — извлекает coords inline, + rating, text_reviews_count, developer_name. +

+
{ + e.preventDefault(); + nbMut.mutate({ slug: nbSlug, id: nbId, city: nbCity }); + }} + > + + + + +
+ +
+ + {/* 4. Valuation */} +
+

4. Valuation (anonymous history)

+

+ Anonymous GET /otsenka-kvartiry-po-adresu-onlayn/ — house meta + до 30 history + items на page. +

+
{ + e.preventDefault(); + valMut.mutate({ + address: valAddress, + offer_category: valCategory, + offer_type: valType, + page: parseInt(valPage, 10), + }); + }} + > + + + + + +
+ +
);