"use client"; import React, { useState } from "react"; import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import "@/components/trade-in/trade-in.css"; import { Topbar } from "@/components/trade-in/Topbar"; import { apiFetch } from "@/lib/api"; import { formatTime, useScraperSettings, useUpdateScraperSetting, ResultPanel, type ScraperSource, type ScheduleSourceKey, type SweepParamConfig, } from "@/app/scrapers/_components/ScraperPage"; import { SystemHealthSection } from "@/components/scrapers/ProxyHealthCard"; import { RunsTable } from "@/components/scrapers/RunsTable"; import { ScheduleControl } from "@/components/scrapers/ScheduleControl"; import { ProviderProxySection } from "@/components/scrapers/ProviderProxySection"; import { PacingSection } from "@/components/scrapers/PacingControl"; import { DataQualitySection } from "@/components/scrapers/DataQualitySection"; // ── Types (provider-specific responses) ─────────────────────────────────── interface ScrapeAroundResp { total_fetched: number; total_inserted: number; total_updated: number; by_source: Array<{ source: string; fetched: number; inserted: number; updated: number; }>; } interface HouseTriggerResp { ok: boolean; house_url: string; counters: Record; } interface DetailTriggerResp { ok: boolean; item_id: string; updated: boolean; } interface ImvResp { ok: boolean; cache_key: string; recommended_price: number; range: [number, number]; market_count: number | null; evaluation_id: number; history_saved: number; } interface CianDetailTriggerResp { ok: boolean; offer_url: string; cian_id: number | null; price_changes_count: number; saved: boolean; listing_id: number | null; } interface CianNewbuildingTriggerResp { ok: boolean; zhk_url: string; cian_internal_house_id: number | null; name: string | null; price_dynamics_count: number; has_management_company: boolean; saved: boolean; house_id: number | null; } interface TestAuthResp { authenticated: boolean; userId: number | null; reason: string | null; } interface UploadCookiesResp { ok?: boolean; userId?: number; cookieCount?: number; detail?: string; } interface CookieEditorEntry { name: string; value: string | number | boolean; [key: string]: unknown; } interface YandexDetailTriggerResp { ok: boolean; offer_url: string; offer_id: string | null; price_rub: number | null; title: string | null; photo_count: number; } interface YandexNewbuildingTriggerResp { ok: boolean; ext_id: string; ext_slug: string; name: string | null; lat: number | null; lon: number | null; rating: number | null; ratings_count: number | null; text_reviews_count: number | null; developer_name: string | null; } interface YandexValuationTriggerResp { ok: boolean; address: string; year_built: number | null; total_floors: number | null; house_type: string | null; has_lift: boolean | null; total_objects: number | null; history_items_count: number; } // ── Backfill / full-load response types ─────────────────────────────────── interface CianBackfillHistoryResp { ok: boolean; listings_total: number; listings_processed: number; listings_succeeded: number; listings_failed_fetch: number; listings_failed_save: number; price_changes_attempted: number; houses_total: number; houses_processed: number; houses_succeeded: number; houses_failed_fetch: number; houses_failed_save: number; valuations_total: number; valuations_processed: number; valuations_succeeded: number; valuations_failed: number; duration_sec: number; } interface BackfillCountersResp { ok: boolean; checked: number; saved: number; skipped: number; errors: number; duration_sec: number; } interface HouseIMVBackfillResp { ok: boolean; checked: number; saved: number; skipped: number; errors: number; duration_sec: number; status_counts: Record; } interface GeocodeStatusResp { total_pending_listings: number; unique_addresses_pending: number; estimated_nominatim_minutes: number; cache_size_active: number; per_source: Array<{ source: string; total: number; pending_geocode: number; has_coords: number; }>; } interface GeocodeBackfillResp { status: string; batch_size: number; dry_run: boolean; addresses_total?: number; addresses_processed?: number; addresses_geocoded?: number; addresses_failed?: number; listings_updated?: number; cache_hits?: number; cache_misses?: number; duration_sec?: number; note?: string; } interface FullLoadStartResp { run_id: number; status: string; pages_per_anchor: number; detail_top_n: number; } interface YandexNBSweepStartResp { run_id: number; status: string; limit: number; detail: string; } // ── Cookie helpers (Cian) ────────────────────────────────────────────────── function convertCookiesFormat(input: string): Record { const parsed: unknown = JSON.parse(input.trim()); if (Array.isArray(parsed)) { return (parsed as CookieEditorEntry[]).reduce>( (acc, c) => { if ( typeof c === "object" && c !== null && "name" in c && "value" in c ) { acc[String(c.name)] = String(c.value); } return acc; }, {}, ); } if (typeof parsed === "object" && parsed !== null && !Array.isArray(parsed)) { return Object.fromEntries( Object.entries(parsed as Record).map(([k, v]) => [ k, String(v), ]), ); } throw new Error("Expected JSON array or plain object"); } function parsePreview(raw: string): { cookies: Record | null; error: string | null; } { if (!raw.trim()) return { cookies: null, error: null }; try { return { cookies: convertCookiesFormat(raw), error: null }; } catch (e) { return { cookies: null, error: e instanceof Error ? e.message : String(e), }; } } // ── Section 2: Глобально ────────────────────────────────────────────────── function GlobalSection() { const settingsQ = useScraperSettings(); const updateMut = useUpdateScraperSetting(); const globalSetting = settingsQ.data?.find((s) => s.source === "global"); const [delayInput, setDelayInput] = useState("0"); const [initialized, setInitialized] = useState(false); if (globalSetting && !initialized) { setDelayInput(String(globalSetting.request_delay_sec)); setInitialized(true); } return (

Глобально

Global delay — нижний порог для всех скрапперов. Изменение вступает в силу немедленно.

{settingsQ.isPending && (

Загрузка настроек…

)} {settingsQ.isError && (

Ошибка загрузки: {settingsQ.error.message}

)} {globalSetting && (
Текущий global delay: {globalSetting.request_delay_sec} сек
{globalSetting.updated_at && (
Обновлено: {formatTime(globalSetting.updated_at)}
)}
)}
{ e.preventDefault(); updateMut.mutate({ source: "global", request_delay_sec: parseFloat(delayInput), }); }} > setDelayInput(e.target.value)} style={{ width: "100%", gridColumn: "1 / -1" }} />
{updateMut.isError && (
Ошибка: {updateMut.error.message}
)} {updateMut.isSuccess && updateMut.data && (
Global delay обновлён:{" "} {Number(updateMut.data.request_delay_sec).toFixed(1)} сек {" "} (обновлено {formatTime(updateMut.data.updated_at)})
)} {/* Per-source delay table (read-only summary) */} {settingsQ.data && settingsQ.data.length > 0 && (

Per-source задержки:

{settingsQ.data.map((s) => ( ))}
Source Delay (сек) Описание Обновлено
{s.source} {s.request_delay_sec.toFixed(1)} {s.description ?? "—"} {formatTime(s.updated_at)}
)}
); } // ── Collapsible wrapper ──────────────────────────────────────────────────── interface CollapsibleProps { title: string; defaultOpen?: boolean; children: React.ReactNode; } function Collapsible({ title, defaultOpen = false, children }: CollapsibleProps) { const [open, setOpen] = useState(defaultOpen); return (
{open &&
{children}
}
); } // ── Avito advanced triggers ──────────────────────────────────────────────── function AvitoAdvancedTriggers() { const [lat, setLat] = useState("56.8400"); const [lon, setLon] = useState("60.6050"); const [radius, setRadius] = useState("1500"); const aroundMut = useMutation< ScrapeAroundResp, Error, { lat: number; lon: number; radius_m: number } >({ mutationFn: (input) => apiFetch("/api/v1/admin/scrape", { method: "POST", body: JSON.stringify({ ...input, sources: ["avito"] }), }), }); const [houseUrl, setHouseUrl] = useState( "/catalog/houses/ekaterinburg/ul_akademika_postovskogo_17a/3171365", ); const houseMut = useMutation({ mutationFn: ({ house_url }) => apiFetch( `/api/v1/admin/scrape/avito-house?house_url=${encodeURIComponent(house_url)}`, { method: "POST" }, ), }); const [itemUrl, setItemUrl] = useState( "/ekaterinburg/kvartiry/3-k._kvartira_75_m_2026_et._7986882804", ); const detailMut = useMutation({ mutationFn: ({ item_url }) => apiFetch( `/api/v1/admin/scrape/avito-detail?item_url=${encodeURIComponent(item_url)}`, { method: "POST" }, ), }); const [imvAddress, setImvAddress] = useState( "Свердловская обл., Екатеринбург, ул. Чайковского, 66А", ); const [imvRooms, setImvRooms] = useState("2"); const [imvArea, setImvArea] = useState("42"); const [imvFloor, setImvFloor] = useState("4"); const [imvTotalFloors, setImvTotalFloors] = useState("5"); const [imvHouseType, setImvHouseType] = useState("panel"); const [imvRenovation, setImvRenovation] = useState("cosmetic"); const [imvBalcony, setImvBalcony] = useState(true); const [imvLoggia, setImvLoggia] = useState(false); const imvMut = useMutation< ImvResp, Error, { address: string; rooms: number; area_m2: number; floor: number; floor_at_home: number; house_type: string; renovation_type: string; has_balcony: boolean; has_loggia: boolean; } >({ mutationFn: (input) => { const qs = new URLSearchParams({ address: input.address, rooms: String(input.rooms), area_m2: String(input.area_m2), floor: String(input.floor), floor_at_home: String(input.floor_at_home), house_type: input.house_type, renovation_type: input.renovation_type, has_balcony: String(input.has_balcony), has_loggia: String(input.has_loggia), }); return apiFetch( `/api/v1/admin/scrape/avito-imv?${qs.toString()}`, { method: "POST" }, ); }, }); return ( <>

1. Search around

{ e.preventDefault(); aroundMut.mutate({ lat: parseFloat(lat), lon: parseFloat(lon), radius_m: parseInt(radius, 10), }); }} >

2. Houses Catalog enrichment

{ e.preventDefault(); houseMut.mutate({ house_url: houseUrl }); }} >

3. Detail page enrichment

{ e.preventDefault(); detailMut.mutate({ item_url: itemUrl }); }} >

4. IMV evaluation (debug)

{ e.preventDefault(); imvMut.mutate({ address: imvAddress, rooms: parseInt(imvRooms, 10), area_m2: parseFloat(imvArea), floor: parseInt(imvFloor, 10), floor_at_home: parseInt(imvTotalFloors, 10), house_type: imvHouseType, renovation_type: imvRenovation, has_balcony: imvBalcony, has_loggia: imvLoggia, }); }} >
); } // ── Cian Auto-Login ──────────────────────────────────────────────────────── interface AutoLoginResp { ok: boolean; userId?: number; cookieCount?: number; } function CianAutoLoginButton() { const qc = useQueryClient(); const [email, setEmail] = useState(""); const [password, setPassword] = useState(""); const [expanded, setExpanded] = useState(false); const autoLoginMut = useMutation({ mutationFn: (body) => apiFetch("/api/v1/admin/scrape/cian/auto-login", { method: "POST", body: body && (body.email || body.password) ? JSON.stringify(body) : undefined, }), onSuccess: () => { void qc.invalidateQueries({ queryKey: ["cian-test-auth"] }); }, }); return (

Авто-логин — браузерный логин через camoufox (требует настроенных{" "} CIAN_LOGIN_EMAIL/PASSWORD в ENV или ввода ниже).

{expanded && (
)}
{autoLoginMut.isSuccess && (
{autoLoginMut.data.ok ? ( <> Залогинились. userId={autoLoginMut.data.userId ?? "—"} · cookieCount={autoLoginMut.data.cookieCount ?? "—"} ) : ( Авто-логин не удался. )}
)} {autoLoginMut.isError && (
Ошибка: {autoLoginMut.error.message}
)}
); } // ── Cian Cookies section ─────────────────────────────────────────────────── function CianCookiesSection() { const [rawInput, setRawInput] = useState(""); const [showInstructions, setShowInstructions] = useState(false); const preview = parsePreview(rawInput); const previewKeys = preview.cookies ? Object.keys(preview.cookies) : []; const uploadMutation = useMutation({ mutationFn: () => { if (!preview.cookies) { throw new Error("Invalid JSON — fix the input first"); } return apiFetch( "/api/v1/admin/scrape/cian/upload-cookies", { method: "POST", body: JSON.stringify(preview.cookies), }, ); }, onSuccess: () => setRawInput(""), }); const testAuthQuery = useQuery({ queryKey: ["cian-test-auth"], queryFn: () => apiFetch("/api/v1/admin/scrape/cian/test-auth"), enabled: false, retry: false, }); const canUpload = !!preview.cookies && previewKeys.length > 0 && !uploadMutation.isPending; return (

Cookies (DMIR_AUTH)

Обновление сессии Cian для скрапера оценщика без перезапуска backend.

{showInstructions && (
  1. Открой{" "} cian.ru {" "} и войди под нужным аккаунтом.
  2. Установи расширение Cookie-Editor (Chrome/Firefox) или DevTools → Application → Cookies → www.cian.ru.
  3. Cookie-Editor: Export → Export as JSON → скопируй массив.
  4. Вставь JSON в поле ниже — формат определится автоматически.
  5. Убедись что preview показывает нужные ключи, нажми Upload.
)}