From dd339d3302494886a29534a7f4e0cde4580a7bd5 Mon Sep 17 00:00:00 2001 From: bot-backend Date: Sat, 20 Jun 2026 09:33:07 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein-frontend):=20=D0=B5=D0=B4=D0=B8?= =?UTF-8?q?=D0=BD=D0=B0=D1=8F=20scrapers-=D1=81=D1=82=D1=80=D0=B0=D0=BD?= =?UTF-8?q?=D0=B8=D1=86=D0=B0=20=E2=80=94=20=D1=82=D0=B0=D0=B1=D1=8B=20Avi?= =?UTF-8?q?to/Cian/Yandex,=20proxy-health+rotate,=20runs-=D0=B8=D1=81?= =?UTF-8?q?=D1=82=D0=BE=D1=80=D0=B8=D1=8F,=20=D0=BA=D0=BE=D0=BD=D1=81?= =?UTF-8?q?=D0=BE=D0=BB=D0=B8=D0=B4=D0=B0=D1=86=D0=B8=D1=8F=20=D1=82=D1=80?= =?UTF-8?q?=D0=B8=D0=B3=D0=B3=D0=B5=D1=80=D0=BE=D0=B2,=20=D1=83=D0=B4?= =?UTF-8?q?=D0=B0=D0=BB=D1=91=D0=BD=20cian-cookies=20=D1=80=D0=B5=D0=B4?= =?UTF-8?q?=D0=B8=D1=80=D0=B5=D0=BA=D1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/app/scrapers/cian-cookies/page.tsx | 21 - .../frontend/src/app/scrapers/page.tsx | 1738 ++++++++++++++--- .../scrapers/ProviderProxySection.tsx | 137 ++ .../components/scrapers/ProxyHealthCard.tsx | 320 +++ .../src/components/scrapers/RunsTable.tsx | 329 ++++ .../components/scrapers/ScheduleControl.tsx | 320 +++ .../src/components/trade-in/Topbar.tsx | 9 +- 7 files changed, 2544 insertions(+), 330 deletions(-) delete mode 100644 tradein-mvp/frontend/src/app/scrapers/cian-cookies/page.tsx create mode 100644 tradein-mvp/frontend/src/components/scrapers/ProviderProxySection.tsx create mode 100644 tradein-mvp/frontend/src/components/scrapers/ProxyHealthCard.tsx create mode 100644 tradein-mvp/frontend/src/components/scrapers/RunsTable.tsx create mode 100644 tradein-mvp/frontend/src/components/scrapers/ScheduleControl.tsx diff --git a/tradein-mvp/frontend/src/app/scrapers/cian-cookies/page.tsx b/tradein-mvp/frontend/src/app/scrapers/cian-cookies/page.tsx deleted file mode 100644 index 6bd4c037..00000000 --- a/tradein-mvp/frontend/src/app/scrapers/cian-cookies/page.tsx +++ /dev/null @@ -1,21 +0,0 @@ -"use client"; - -import { useEffect } from "react"; -import { useRouter } from "next/navigation"; - -/** - * /scrapers/cian-cookies redirects to /scrapers/cian where cookie management - * is now integrated as a dedicated section at the bottom of the Cian page. - */ -export default function CianCookiesRedirect() { - const router = useRouter(); - useEffect(() => { - router.replace("/scrapers/cian"); - }, [router]); - - return ( -

- Перенаправление на страницу Cian скраппера… -

- ); -} diff --git a/tradein-mvp/frontend/src/app/scrapers/page.tsx b/tradein-mvp/frontend/src/app/scrapers/page.tsx index fdbe7158..1c42147c 100644 --- a/tradein-mvp/frontend/src/app/scrapers/page.tsx +++ b/tradein-mvp/frontend/src/app/scrapers/page.tsx @@ -1,25 +1,182 @@ "use client"; -import { useEffect, useState } from "react"; +import React, { useState } from "react"; +import { useMutation, useQuery } from "@tanstack/react-query"; import "@/components/trade-in/trade-in.css"; import { Topbar } from "@/components/trade-in/Topbar"; -import { API_BASE_URL } from "@/lib/api"; +import { apiFetch } from "@/lib/api"; import { formatTime, - translateStatus, useScraperSettings, - useSchedules, useUpdateScraperSetting, - useUpdateSchedule, - utcToMsk, - type ScheduleConfig, + ResultPanel, + type ScraperSource, type ScheduleSourceKey, + type SweepParamConfig, } from "@/app/scrapers/_components/ScraperPage"; +import { SystemHealthSection } from "@/components/scrapers/ProxyHealthCard"; +import { RunsTable } from "@/components/scrapers/RunsTable"; +import { ScheduleControl } from "@/components/scrapers/ScheduleControl"; +import { ProviderProxySection } from "@/components/scrapers/ProviderProxySection"; -// ── Global delay section ─────────────────────────────────────────────────── +// ── Types (provider-specific responses) ─────────────────────────────────── -function GlobalDelaySetting() { +interface ScrapeAroundResp { + total_fetched: number; + total_inserted: number; + total_updated: number; + by_source: Array<{ + source: string; + fetched: number; + inserted: number; + updated: number; + }>; +} + +interface HouseTriggerResp { + ok: boolean; + house_url: string; + counters: Record; +} + +interface DetailTriggerResp { + ok: boolean; + item_id: string; + updated: boolean; +} + +interface ImvResp { + ok: boolean; + cache_key: string; + recommended_price: number; + range: [number, number]; + market_count: number | null; + evaluation_id: number; + history_saved: number; +} + +interface CianDetailTriggerResp { + ok: boolean; + offer_url: string; + cian_id: number | null; + price_changes_count: number; + saved: boolean; + listing_id: number | null; +} + +interface CianNewbuildingTriggerResp { + ok: boolean; + zhk_url: string; + cian_internal_house_id: number | null; + name: string | null; + price_dynamics_count: number; + has_management_company: boolean; + saved: boolean; + house_id: number | null; +} + +interface TestAuthResp { + authenticated: boolean; + userId: number | null; + reason: string | null; +} + +interface UploadCookiesResp { + ok?: boolean; + userId?: number; + cookieCount?: number; + detail?: string; +} + +interface CookieEditorEntry { + name: string; + value: string | number | boolean; + [key: string]: unknown; +} + +interface YandexDetailTriggerResp { + ok: boolean; + offer_url: string; + offer_id: string | null; + price_rub: number | null; + title: string | null; + photo_count: number; +} + +interface YandexNewbuildingTriggerResp { + ok: boolean; + ext_id: string; + ext_slug: string; + name: string | null; + lat: number | null; + lon: number | null; + rating: number | null; + ratings_count: number | null; + text_reviews_count: number | null; + developer_name: string | null; +} + +interface YandexValuationTriggerResp { + ok: boolean; + address: string; + year_built: number | null; + total_floors: number | null; + house_type: string | null; + has_lift: boolean | null; + total_objects: number | null; + history_items_count: number; +} + +// ── Cookie helpers (Cian) ────────────────────────────────────────────────── + +function convertCookiesFormat(input: string): Record { + const parsed: unknown = JSON.parse(input.trim()); + if (Array.isArray(parsed)) { + return (parsed as CookieEditorEntry[]).reduce>( + (acc, c) => { + if ( + typeof c === "object" && + c !== null && + "name" in c && + "value" in c + ) { + acc[String(c.name)] = String(c.value); + } + return acc; + }, + {}, + ); + } + if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) { + return Object.fromEntries( + Object.entries(parsed as Record).map(([k, v]) => [ + k, + String(v), + ]), + ); + } + throw new Error("Expected JSON array or plain object"); +} + +function parsePreview(raw: string): { + cookies: Record | null; + error: string | null; +} { + if (!raw.trim()) return { cookies: null, error: null }; + try { + return { cookies: convertCookiesFormat(raw), error: null }; + } catch (e) { + return { + cookies: null, + error: e instanceof Error ? e.message : String(e), + }; + } +} + +// ── Section 2: Глобально ────────────────────────────────────────────────── + +function GlobalSection() { const settingsQ = useScraperSettings(); const updateMut = useUpdateScraperSetting(); @@ -27,36 +184,34 @@ function GlobalDelaySetting() { const [delayInput, setDelayInput] = useState("0"); const [initialized, setInitialized] = useState(false); - useEffect(() => { - if (globalSetting && !initialized) { - setDelayInput(String(globalSetting.request_delay_sec)); - setInitialized(true); - } - }, [globalSetting, initialized]); + if (globalSetting && !initialized) { + setDelayInput(String(globalSetting.request_delay_sec)); + setInitialized(true); + } return (
-

Глобальная задержка запросов (нижний порог)

+

Глобально

- Глобальный делэй — нижний порог для всех скрапперов (max с per-source - defaults). Если 0 — только per-source defaults. Изменение вступает в - силу немедленно без перезапуска процесса. Для per-source задержек - зайдите на страницу конкретного скраппера. + Global delay — нижний порог для всех скрапперов. Изменение вступает + в силу немедленно.

{settingsQ.isPending && (

Загрузка настроек…

)} - {settingsQ.error && ( + {settingsQ.isError && (

Ошибка загрузки: {settingsQ.error.message}

)} {globalSetting && ( -
+
- Текущий global delay: + + Текущий global delay: + {globalSetting.request_delay_sec} сек @@ -70,60 +225,6 @@ function GlobalDelaySetting() {
)} - {/* Per-source read-only table */} - {settingsQ.data && settingsQ.data.length > 0 && ( -
-

- Все per-source задержки (read-only здесь, редактировать на - страницах провайдеров): -

- - - - - - - - - - - {settingsQ.data.map((s) => ( - - - - - - - ))} - -
SourceDelay (сек)ОписаниеОбновлено
- {s.source} - {s.request_delay_sec.toFixed(1)} - {s.description ?? "—"} - - {formatTime(s.updated_at)} -
-
- )} -
{ @@ -163,7 +264,7 @@ function GlobalDelaySetting() {
- {updateMut.error && ( + {updateMut.isError && (
Ошибка: {updateMut.error.message}
@@ -177,246 +278,1281 @@ function GlobalDelaySetting() { (обновлено {formatTime(updateMut.data.updated_at)})
)} -
- ); -} -// ── Schedules summary table ──────────────────────────────────────────────── - -const SCHEDULE_SOURCE_LABELS: Record = { - avito_city_sweep: "Avito city sweep", - cian_city_sweep: "Cian city sweep", - yandex_city_sweep: "Yandex city sweep", -}; - -const SCHEDULE_SOURCES: ScheduleSourceKey[] = [ - "avito_city_sweep", - "cian_city_sweep", - "yandex_city_sweep", -]; - -function SchedulesSummary() { - const schedulesQ = useSchedules(); - const updateMut = useUpdateSchedule(); - - function handleToggle(schedule: ScheduleConfig) { - updateMut.mutate({ - source: schedule.source as ScheduleSourceKey, - payload: { - enabled: !schedule.enabled, - window_start_hour: schedule.window_start_hour, - window_end_hour: schedule.window_end_hour, - default_params: schedule.default_params as Record, - }, - }); - } - - // Build ordered rows from known sources + any extras - const knownRows = SCHEDULE_SOURCES.map((src) => - schedulesQ.data?.find((s) => s.source === src), - ).filter((s): s is ScheduleConfig => s !== undefined); - - const extraRows = - schedulesQ.data?.filter( - (s) => !SCHEDULE_SOURCES.includes(s.source as ScheduleSourceKey), - ) ?? []; - - const rows = [...knownRows, ...extraRows]; - - return ( -
-

Сводная таблица расписаний

-

- Быстрое включение/отключение всех cron-расписаний. Параметры и окна - настраиваются на страницах провайдеров. Автообновление каждые 30 сек. -

- - {schedulesQ.isPending &&

Загрузка…

} - {schedulesQ.error && ( -

- Ошибка загрузки: {schedulesQ.error.message} -

- )} - - {rows.length > 0 && ( - - - - - - - - - - - - - - {rows.map((s) => { - const mskStart = utcToMsk(s.window_start_hour); - const mskEnd = utcToMsk(s.window_end_hour); - const label = - SCHEDULE_SOURCE_LABELS[s.source] ?? s.source; - return ( - - - - - - - - - - ); - })} - -
SourceСтатусОкно (МСК)Следующий запускПоследний запускПоследний runВкл/выкл
- {label} - - - {s.enabled ? "включено" : "отключено"} - - - {String(mskStart).padStart(2, "0")}:00– - {String(mskEnd).padStart(2, "0")}:00 МСК - {formatTime(s.next_run_at)}{formatTime(s.last_run_at)} - {s.last_run_id ? ( - #{s.last_run_id} - ) : ( - "—" - )} - - -
- )} - - {rows.length === 0 && !schedulesQ.isPending && !schedulesQ.error && ( -

Расписания не найдены. Они создаются автоматически при первом сохранении на странице провайдера.

- )} - - {updateMut.error && ( -
- Ошибка обновления: {updateMut.error.message} -
- )} - {updateMut.isSuccess && ( -
- Расписание {translateStatus( - updateMut.data.enabled ? "running" : "cancelled", - )}. -
- )} -
- ); -} - -// ── Provider cards ───────────────────────────────────────────────────────── - -const PROVIDERS = [ - { - key: "avito", - name: "Avito", - href: "/scrapers/avito", - description: - "City sweep, houses catalog, detail enrichment, IMV evaluation", - }, - { - key: "cian", - name: "Cian", - href: "/scrapers/cian", - description: "City sweep, detail enrichment, новостройки ЖК, cookies", - }, - { - key: "yandex", - name: "Yandex Realty", - href: "/scrapers/yandex", - description: "City sweep, detail, ЖК landing, valuation", - }, -]; - -function ProviderCards() { - return ( -
-

Провайдеры

-
- {PROVIDERS.map((p) => ( - 0 && ( + + Per-source задержки: +

+ + + + + + + + + + + {settingsQ.data.map((s) => ( + + + + + + + ))} + +
SourceDelay (сек)ОписаниеОбновлено
+ {s.source} + {s.request_delay_sec.toFixed(1)} + {s.description ?? "—"} + + {formatTime(s.updated_at)} +
+
+ )}
); } +// ── Collapsible wrapper ──────────────────────────────────────────────────── + +interface CollapsibleProps { + title: string; + defaultOpen?: boolean; + children: React.ReactNode; +} + +function Collapsible({ title, defaultOpen = false, children }: CollapsibleProps) { + const [open, setOpen] = useState(defaultOpen); + return ( +
+ + {open &&
{children}
} +
+ ); +} + +// ── Avito advanced triggers ──────────────────────────────────────────────── + +function AvitoAdvancedTriggers() { + const [lat, setLat] = useState("56.8400"); + const [lon, setLon] = useState("60.6050"); + const [radius, setRadius] = useState("1500"); + const aroundMut = useMutation< + ScrapeAroundResp, + Error, + { lat: number; lon: number; radius_m: number } + >({ + mutationFn: (input) => + apiFetch("/api/v1/admin/scrape", { + method: "POST", + body: JSON.stringify({ ...input, sources: ["avito"] }), + }), + }); + + const [houseUrl, setHouseUrl] = useState( + "/catalog/houses/ekaterinburg/ul_akademika_postovskogo_17a/3171365", + ); + const houseMut = useMutation({ + mutationFn: ({ house_url }) => + apiFetch( + `/api/v1/admin/scrape/avito-house?house_url=${encodeURIComponent(house_url)}`, + { method: "POST" }, + ), + }); + + const [itemUrl, setItemUrl] = useState( + "/ekaterinburg/kvartiry/3-k._kvartira_75_m_2026_et._7986882804", + ); + const detailMut = useMutation({ + mutationFn: ({ item_url }) => + apiFetch( + `/api/v1/admin/scrape/avito-detail?item_url=${encodeURIComponent(item_url)}`, + { method: "POST" }, + ), + }); + + const [imvAddress, setImvAddress] = useState( + "Свердловская обл., Екатеринбург, ул. Чайковского, 66А", + ); + const [imvRooms, setImvRooms] = useState("2"); + const [imvArea, setImvArea] = useState("42"); + const [imvFloor, setImvFloor] = useState("4"); + const [imvTotalFloors, setImvTotalFloors] = useState("5"); + const [imvHouseType, setImvHouseType] = useState("panel"); + const [imvRenovation, setImvRenovation] = useState("cosmetic"); + const [imvBalcony, setImvBalcony] = useState(true); + const [imvLoggia, setImvLoggia] = useState(false); + const imvMut = useMutation< + ImvResp, + Error, + { + address: string; + rooms: number; + area_m2: number; + floor: number; + floor_at_home: number; + house_type: string; + renovation_type: string; + has_balcony: boolean; + has_loggia: boolean; + } + >({ + mutationFn: (input) => { + const qs = new URLSearchParams({ + address: input.address, + rooms: String(input.rooms), + area_m2: String(input.area_m2), + floor: String(input.floor), + floor_at_home: String(input.floor_at_home), + house_type: input.house_type, + renovation_type: input.renovation_type, + has_balcony: String(input.has_balcony), + has_loggia: String(input.has_loggia), + }); + return apiFetch( + `/api/v1/admin/scrape/avito-imv?${qs.toString()}`, + { method: "POST" }, + ); + }, + }); + + return ( + <> +
+

1. Search around

+
{ + e.preventDefault(); + aroundMut.mutate({ + lat: parseFloat(lat), + lon: parseFloat(lon), + radius_m: parseInt(radius, 10), + }); + }} + > + + + + +
+ +
+ +
+

2. Houses Catalog enrichment

+
{ + e.preventDefault(); + houseMut.mutate({ house_url: houseUrl }); + }} + > + + +
+ +
+ +
+

3. Detail page enrichment

+
{ + e.preventDefault(); + detailMut.mutate({ item_url: itemUrl }); + }} + > + + +
+ +
+ +
+

4. IMV evaluation (debug)

+
{ + e.preventDefault(); + imvMut.mutate({ + address: imvAddress, + rooms: parseInt(imvRooms, 10), + area_m2: parseFloat(imvArea), + floor: parseInt(imvFloor, 10), + floor_at_home: parseInt(imvTotalFloors, 10), + house_type: imvHouseType, + renovation_type: imvRenovation, + has_balcony: imvBalcony, + has_loggia: imvLoggia, + }); + }} + > + + + + + + + + + + +
+ +
+ + ); +} + +// ── Cian Cookies section ─────────────────────────────────────────────────── + +function CianCookiesSection() { + const [rawInput, setRawInput] = useState(""); + const [showInstructions, setShowInstructions] = useState(false); + + const preview = parsePreview(rawInput); + const previewKeys = preview.cookies ? Object.keys(preview.cookies) : []; + + const uploadMutation = useMutation({ + mutationFn: () => { + if (!preview.cookies) { + throw new Error("Invalid JSON — fix the input first"); + } + return apiFetch( + "/api/v1/admin/scrape/cian/upload-cookies", + { + method: "POST", + body: JSON.stringify(preview.cookies), + }, + ); + }, + onSuccess: () => setRawInput(""), + }); + + const testAuthQuery = useQuery({ + queryKey: ["cian-test-auth"], + queryFn: () => + apiFetch("/api/v1/admin/scrape/cian/test-auth"), + enabled: false, + retry: false, + }); + + const canUpload = + !!preview.cookies && previewKeys.length > 0 && !uploadMutation.isPending; + + return ( +
+

Cookies (DMIR_AUTH)

+

+ Обновление сессии Cian для скрапера оценщика без перезапуска backend. +

+ + + {showInstructions && ( +
    +
  1. + Открой{" "} + + cian.ru + {" "} + и войди под нужным аккаунтом. +
    2. +
  2. + Установи расширение Cookie-Editor (Chrome/Firefox) + или DevTools → Application → Cookies → www.cian.ru. +
    3. +
  3. + Cookie-Editor: Export → Export as JSON → скопируй массив. +
    4. +
  4. Вставь JSON в поле ниже — формат определится автоматически.
    5. +
  5. Убедись что preview показывает нужные ключи, нажми Upload.
    6. +
+ )} + +
+