From 461161e3ecf52a21cbc0e74855c68e52757290d5 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 18:39:39 +0300 Subject: [PATCH 1/2] =?UTF-8?q?feat(tradein-ui):=20Yandex=20admin=20scrape?= =?UTF-8?q?r=20page=20=E2=80=94=20mirror=20Avito=20+=20global=20delay=20co?= =?UTF-8?q?ntrol?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the 'В разработке' stub at /scrapers/yandex with a full page mirroring the Avito admin pattern, adapted for the 4 Yandex scrapers landed in PRs #453/#456/#462/#466/#467/#468 + admin endpoints from #484. Sections: - Global delay (top, highlighted) — GET/PUT /api/v1/admin/scraper-settings, one knob applies to all 4 Yandex scrapers via umbrella 'yandex' key - 1. Search around — POST /api/v1/admin/scrape with sources=['yandex'] + multi_room_yandex / deep_yandex toggles - 2. Detail — POST /api/v1/admin/scrape/yandex-detail?offer_url= (snapshot) - 3. JK Newbuilding — POST /api/v1/admin/scrape/yandex-newbuilding?slug=&id=&city= - 4. Valuation — POST /api/v1/admin/scrape/yandex-valuation?address=&offer_category=&offer_type=&page= Skipped vs Avito: city-sweep + schedule sections — no Yandex cron/sweep job exists yet (PR follow-up if needed). Defaults: EKB lat 56.84 / lon 60.605, JK Tatlin (tatlin/1592987), Uchiteley 18. TypeScript strict, reuses trade-in.css classes, TanStack Query hooks + apiFetch. ResultPanel inlined. --- .../frontend/src/app/scrapers/yandex/page.tsx | 557 +++++++++++++++++- 1 file changed, 548 insertions(+), 9 deletions(-) diff --git a/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx b/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx index 47506196..cfb6dd29 100644 --- a/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx +++ b/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx @@ -1,23 +1,562 @@ "use client"; +import { useEffect, useState } from "react"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + import "@/components/trade-in/trade-in.css"; import { Topbar } from "@/components/trade-in/Topbar"; +import { apiFetch } from "@/lib/api"; + +// ── Types ────────────────────────────────────────────────────────────────── + +interface ScraperSetting { + source: string; + request_delay_sec: number; + description: string | null; + updated_at: string | null; +} + +interface ScraperSettingUpdate { + request_delay_sec: number; +} + +interface ScrapeAroundInput { + lat: number; + lon: number; + radius_m: number; + sources: string[]; + multi_room_yandex?: boolean; + deep_yandex?: boolean; +} + +interface ScrapeAroundResp { + total_fetched: number; + total_inserted: number; + total_updated: number; + by_source: Array<{ + source: string; + fetched: number; + inserted: number; + updated: number; + }>; +} + +interface YandexDetailTriggerResp { + ok: boolean; + offer_url: string; + offer_id: string | null; + price_rub: number | null; + title: string | null; + photo_count: number; +} + +interface YandexNewbuildingTriggerResp { + ok: boolean; + ext_id: string; + ext_slug: string; + name: string | null; + lat: number | null; + lon: number | null; + rating: number | null; + ratings_count: number | null; + text_reviews_count: number | null; + developer_name: string | null; +} + +interface YandexValuationTriggerResp { + ok: boolean; + address: string; + year_built: number | null; + total_floors: number | null; + house_type: string | null; + has_lift: boolean | null; + total_objects: number | null; + history_items_count: number; +} + +// ── Hooks ────────────────────────────────────────────────────────────────── + +function useScraperSettings() { + return useQuery({ + queryKey: ["scraper-settings"], + queryFn: () => apiFetch("/api/v1/admin/scraper-settings"), + refetchInterval: 30_000, + }); +} + +function useUpdateScraperSetting() { + const qc = useQueryClient(); + return useMutation< + ScraperSetting, + Error, + { source: string; payload: ScraperSettingUpdate } + >({ + mutationFn: ({ source, payload }) => + apiFetch(`/api/v1/admin/scraper-settings/${source}`, { + method: "PUT", + body: JSON.stringify(payload), + }), + onSuccess: () => { + qc.invalidateQueries({ queryKey: ["scraper-settings"] }); + }, + }); +} + +function useScrapeAround() { + return useMutation({ + mutationFn: (input) => + apiFetch("/api/v1/admin/scrape", { + method: "POST", + body: JSON.stringify(input), + }), + }); +} + +function useScrapeYandexDetail() { + return useMutation({ + mutationFn: ({ offer_url }) => + apiFetch( + `/api/v1/admin/scrape/yandex-detail?offer_url=${encodeURIComponent(offer_url)}`, + { method: "POST" }, + ), + }); +} + +function useScrapeYandexNewbuilding() { + return useMutation< + YandexNewbuildingTriggerResp, + Error, + { slug: string; id: string; city: string } + >({ + mutationFn: ({ slug, id, city }) => { + const qs = new URLSearchParams({ slug, id, city }); + return apiFetch( + `/api/v1/admin/scrape/yandex-newbuilding?${qs.toString()}`, + { method: "POST" }, + ); + }, + }); +} + +function useScrapeYandexValuation() { + return useMutation< + YandexValuationTriggerResp, + Error, + { address: string; offer_category: string; offer_type: string; page: number } + >({ + mutationFn: ({ address, offer_category, offer_type, page }) => { + const qs = new URLSearchParams({ + address, + offer_category, + offer_type, + page: String(page), + }); + return apiFetch( + `/api/v1/admin/scrape/yandex-valuation?${qs.toString()}`, + { method: "POST" }, + ); + }, + }); +} + +// ── Result panel ─────────────────────────────────────────────────────────── + +interface ResultPanelProps { + mut: { + data?: TData; + error: Error | null; + isPending: boolean; + isSuccess: boolean; + }; +} + +function ResultPanel({ mut }: ResultPanelProps) { + if (mut.error) { + return ( +
+ Ошибка: {mut.error.message} +
+ ); + } + if (mut.isSuccess && mut.data) { + return ( +
+ 
{JSON.stringify(mut.data, null, 2)}
+
+ ); + } + return null; +} + +// ── Helpers ──────────────────────────────────────────────────────────────── + +function formatTime(iso: string | null): string { + if (!iso) return "—"; + try { + return new Date(iso).toLocaleString("ru-RU", { + day: "2-digit", + month: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }); + } catch { + return iso; + } +} + +// ── Page ─────────────────────────────────────────────────────────────────── export default function YandexScraperPage() { + // Settings + const settingsQ = useScraperSettings(); + const updateSettingMut = useUpdateScraperSetting(); + const [delayInput, setDelayInput] = useState("5.0"); + + useEffect(() => { + const yandex = settingsQ.data?.find((s) => s.source === "yandex"); + if (yandex) setDelayInput(String(yandex.request_delay_sec)); + }, [settingsQ.data]); + + // Around + const [lat, setLat] = useState("56.8400"); + const [lon, setLon] = useState("60.6050"); + const [radius, setRadius] = useState("1500"); + const [multiRoom, setMultiRoom] = useState(false); + const [deep, setDeep] = useState(false); + const aroundMut = useScrapeAround(); + + // Detail + const [detailUrl, setDetailUrl] = useState( + "https://realty.yandex.ru/offer/7567094292504417257/", + ); + const detailMut = useScrapeYandexDetail(); + + // Newbuilding + const [nbSlug, setNbSlug] = useState("tatlin"); + const [nbId, setNbId] = useState("1592987"); + const [nbCity, setNbCity] = useState("ekaterinburg"); + const nbMut = useScrapeYandexNewbuilding(); + + // Valuation + const [valAddress, setValAddress] = useState( + "Свердловская область, Екатеринбург, улица Учителей, 18", + ); + const [valCategory, setValCategory] = useState("APARTMENT"); + const [valType, setValType] = useState("SELL"); + const [valPage, setValPage] = useState("1"); + const valMut = useScrapeYandexValuation(); + + const yandexSetting = settingsQ.data?.find((s) => s.source === "yandex"); + return ( <>
-

Yandex скрапер

-

В разработке.

-
-

- Yandex SERP refactor merged (PR #462), но admin UI ещё не подключён. - Используй cron-scrape.sh или прямой curl к{" "} - POST /api/v1/admin/scrape с{" "} - sources: ["yandex"]. +

Yandex скрапер — admin триггер

+

+ Ручной запуск sub-pipeline'ов YandexRealtyScraper v1 + глобальная настройка + задержки между запросами. +

+ + {/* 0. Global delay */} +
+

Глобальная задержка запросов

+

+ Настройка применяется ко ВСЕМ Yandex-скрейперам (SERP, detail, ЖК, valuation) + при создании следующего инстанса (in-process cache 60s). Меняется без рестарта + backend.

-
+ + {settingsQ.isPending &&

Загрузка настроек…

} + {settingsQ.error && ( +

+ Ошибка загрузки настроек: {settingsQ.error.message} +

+ )} + + {yandexSetting && ( +
+
+ Текущая задержка: + + {yandexSetting.request_delay_sec} сек + +
+ {yandexSetting.updated_at && ( +
+ Обновлено: + {formatTime(yandexSetting.updated_at)} +
+ )} + {yandexSetting.description && ( +
+ Описание: + {yandexSetting.description} +
+ )} +
+ )} + +
{ + e.preventDefault(); + updateSettingMut.mutate({ + source: "yandex", + payload: { request_delay_sec: parseFloat(delayInput) }, + }); + }} + > + + +
+ + {updateSettingMut.error && ( +
+ Ошибка: {updateSettingMut.error.message} +
+ )} + {updateSettingMut.isSuccess && updateSettingMut.data && ( +
+ Сохранено. Новая задержка:{" "} + {updateSettingMut.data.request_delay_sec} сек +
+ )} + + + {/* 1. Around point */} +
+

1. Search around (общий /admin/scrape)

+

+ Yandex SERP path-based vtorichka — lat/lon/radius логируется, scraper берёт{" "} + /{ekaterinburg}/kupit/kvartira/vtorichniy-rynok/. + multi-room split не реализован для DOM-парсера (запросы будут одинаковыми). + Deep пробует pages × sorts комбинации. +

+
{ + e.preventDefault(); + aroundMut.mutate({ + lat: parseFloat(lat), + lon: parseFloat(lon), + radius_m: parseInt(radius, 10), + sources: ["yandex"], + multi_room_yandex: multiRoom, + deep_yandex: deep, + }); + }} + > + + + + + + +
+ +
+ + {/* 2. Detail */} +
+

2. Detail page enrichment

+

+ Product JSON-LD price (exact int) + DOM-секции description/agency/metro/photos. +

+
{ + e.preventDefault(); + detailMut.mutate({ offer_url: detailUrl }); + }} + > + + +
+ +
+ + {/* 3. ЖК Newbuilding */} +
+

3. ЖК (Newbuilding) landing

+

+ URL /{city}/kupit/novostrojka/{slug}-{id}/ — извлекает coords inline, + rating, text_reviews_count, developer_name. +

+
{ + e.preventDefault(); + nbMut.mutate({ slug: nbSlug, id: nbId, city: nbCity }); + }} + > + + + + +
+ +
+ + {/* 4. Valuation */} +
+

4. Valuation (anonymous history)

+

+ Anonymous GET /otsenka-kvartiry-po-adresu-onlayn/ — house meta + до 30 history + items на page. +

+
{ + e.preventDefault(); + valMut.mutate({ + address: valAddress, + offer_category: valCategory, + offer_type: valType, + page: parseInt(valPage, 10), + }); + }} + > + + + + + +
+ +
); -- 2.45.3 From 271b5daeeaf914d1219a03e8ae54982db324eb1a Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 23:01:50 +0300 Subject: [PATCH 2/2] =?UTF-8?q?fix(tradein-ui):=20#488=20BLOCK=20=E2=80=94?= =?UTF-8?q?=20scraper-settings=20response=20wrapper=20+=20PUT=20source=20f?= =?UTF-8?q?ield?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit deep-code-reviewer caught 2 critical bugs in both yandex/page.tsx (#488) and cian/page.tsx (#486 already merged): 1. useScraperSettings types response as ScraperSetting[] but backend returns {settings: [...]} -> .find is not a function -> settings panel never renders, slider input never auto-populates from DB. Fix: unwrap .settings in queryFn. 2. PUT body missing required `source` field -- backend Pydantic ScraperSettingPayload validates source==path source. Result: 422 on every save. Fix: include {source, ...payload} in PUT body. Plus M (non-blocking): useEffect race -- refetch every 30s clobbered user input mid-edit. Guarded with initializedRef first-load-only flag. Refs PR #488 review (deep-code-reviewer, 2026-05-23). --- .../frontend/src/app/scrapers/yandex/page.tsx | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx b/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx index cfb6dd29..ef9059c9 100644 --- a/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx +++ b/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx @@ -1,6 +1,6 @@ "use client"; -import { useEffect, useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import "@/components/trade-in/trade-in.css"; @@ -17,6 +17,7 @@ interface ScraperSetting { } interface ScraperSettingUpdate { + source: string; request_delay_sec: number; } @@ -79,7 +80,10 @@ interface YandexValuationTriggerResp { function useScraperSettings() { return useQuery({ queryKey: ["scraper-settings"], - queryFn: () => apiFetch("/api/v1/admin/scraper-settings"), + queryFn: () => + apiFetch<{ settings: ScraperSetting[] }>("/api/v1/admin/scraper-settings").then( + (r) => r.settings, + ), refetchInterval: 30_000, }); } @@ -89,12 +93,12 @@ function useUpdateScraperSetting() { return useMutation< ScraperSetting, Error, - { source: string; payload: ScraperSettingUpdate } + { source: string; payload: Omit } >({ mutationFn: ({ source, payload }) => apiFetch(`/api/v1/admin/scraper-settings/${source}`, { method: "PUT", - body: JSON.stringify(payload), + body: JSON.stringify({ source, ...payload }), }), onSuccess: () => { qc.invalidateQueries({ queryKey: ["scraper-settings"] }); @@ -212,10 +216,14 @@ export default function YandexScraperPage() { const settingsQ = useScraperSettings(); const updateSettingMut = useUpdateScraperSetting(); const [delayInput, setDelayInput] = useState("5.0"); + const delayInitializedRef = useRef(false); useEffect(() => { const yandex = settingsQ.data?.find((s) => s.source === "yandex"); - if (yandex) setDelayInput(String(yandex.request_delay_sec)); + if (yandex && !delayInitializedRef.current) { + setDelayInput(String(yandex.request_delay_sec)); + delayInitializedRef.current = true; + } }, [settingsQ.data]); // Around -- 2.45.3