diff --git a/tradein-mvp/frontend/src/app/scrapers/page.tsx b/tradein-mvp/frontend/src/app/scrapers/page.tsx index 1c42147c..ac466fb6 100644 --- a/tradein-mvp/frontend/src/app/scrapers/page.tsx +++ b/tradein-mvp/frontend/src/app/scrapers/page.tsx @@ -1,7 +1,7 @@ "use client"; import React, { useState } from "react"; -import { useMutation, useQuery } from "@tanstack/react-query"; +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import "@/components/trade-in/trade-in.css"; import { Topbar } from "@/components/trade-in/Topbar"; @@ -19,6 +19,8 @@ import { SystemHealthSection } from "@/components/scrapers/ProxyHealthCard"; import { RunsTable } from "@/components/scrapers/RunsTable"; import { ScheduleControl } from "@/components/scrapers/ScheduleControl"; import { ProviderProxySection } from "@/components/scrapers/ProviderProxySection"; +import { PacingSection } from "@/components/scrapers/PacingControl"; +import { DataQualitySection } from "@/components/scrapers/DataQualitySection"; // ── Types (provider-specific responses) ─────────────────────────────────── @@ -128,6 +130,89 @@ interface YandexValuationTriggerResp { history_items_count: number; } +// ── Backfill / full-load response types ─────────────────────────────────── + +interface CianBackfillHistoryResp { + ok: boolean; + listings_total: number; + listings_processed: number; + listings_succeeded: number; + listings_failed_fetch: number; + listings_failed_save: number; + price_changes_attempted: number; + houses_total: number; + houses_processed: number; + houses_succeeded: number; + houses_failed_fetch: number; + houses_failed_save: number; + valuations_total: number; + valuations_processed: number; + valuations_succeeded: number; + valuations_failed: number; + duration_sec: number; +} + +interface BackfillCountersResp { + ok: boolean; + checked: number; + saved: number; + skipped: number; + errors: number; + duration_sec: number; +} + +interface HouseIMVBackfillResp { + ok: boolean; + checked: number; + saved: number; + skipped: number; + errors: number; + duration_sec: number; + status_counts: Record; +} + +interface GeocodeStatusResp { + total_pending_listings: number; + unique_addresses_pending: number; + estimated_nominatim_minutes: number; + cache_size_active: number; + per_source: Array<{ + source: string; + total: number; + pending_geocode: number; + has_coords: number; + }>; +} + +interface GeocodeBackfillResp { + status: string; + batch_size: number; + dry_run: boolean; + addresses_total?: number; + addresses_processed?: number; + addresses_geocoded?: number; + addresses_failed?: number; + listings_updated?: number; + cache_hits?: number; + cache_misses?: number; + duration_sec?: number; + note?: string; +} + +interface FullLoadStartResp { + run_id: number; + status: string; + pages_per_anchor: number; + detail_top_n: number; +} + +interface YandexNBSweepStartResp { + run_id: number; + status: string; + limit: number; + detail: string; +} + // ── Cookie helpers (Cian) ────────────────────────────────────────────────── function convertCookiesFormat(input: string): Record { @@ -617,6 +702,123 @@ function AvitoAdvancedTriggers() { ); } +// ── Cian Auto-Login ──────────────────────────────────────────────────────── + +interface AutoLoginResp { + ok: boolean; + userId?: number; + cookieCount?: number; +} + +function CianAutoLoginButton() { + const qc = useQueryClient(); + const [email, setEmail] = useState(""); + const [password, setPassword] = useState(""); + const [expanded, setExpanded] = useState(false); + + const autoLoginMut = useMutation({ + mutationFn: (body) => + apiFetch("/api/v1/admin/scrape/cian/auto-login", { + method: "POST", + body: body && (body.email || body.password) ? JSON.stringify(body) : undefined, + }), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ["cian-test-auth"] }); + }, + }); + + return ( +
+

+ Авто-логин — браузерный логин через camoufox (требует настроенных{" "} + CIAN_LOGIN_EMAIL/PASSWORD в ENV или ввода ниже). +

+ + {expanded && ( +
+ + +
+ )} +
+ +
+ {autoLoginMut.isSuccess && ( +
+ {autoLoginMut.data.ok ? ( + <> + Залогинились. userId={autoLoginMut.data.userId ?? "—"} · + cookieCount={autoLoginMut.data.cookieCount ?? "—"} + + ) : ( + Авто-логин не удался. + )} +
+ )} + {autoLoginMut.isError && ( +
+ Ошибка: {autoLoginMut.error.message} +
+ )} +
+ ); +} + // ── Cian Cookies section ─────────────────────────────────────────────────── function CianCookiesSection() { @@ -838,6 +1040,9 @@ function CianCookiesSection() { )} )} + + {/* Auto-login (Variant B) */} + ); } @@ -1055,6 +1260,219 @@ function CianAdvancedTriggers() { Вызывается автоматически из /estimate flow (Stage 9). Admin trigger будет добавлен позже.

+ + + + ); +} + +// ── Cian Backfill section ───────────────────────────────────────────────── + +function CianBackfillSection() { + // ── cian-backfill-history ────────────────────────────────────────────── + const [bfBatchSize, setBfBatchSize] = useState("50"); + const [bfDoListings, setBfDoListings] = useState(true); + const [bfDoHouses, setBfDoHouses] = useState(true); + const [bfDoValuations, setBfDoValuations] = useState(false); + const [bfDryRun, setBfDryRun] = useState(false); + + const backfillHistoryMut = useMutation({ + mutationFn: () => { + const qs = new URLSearchParams({ + batch_size: bfBatchSize, + do_listings: String(bfDoListings), + do_houses: String(bfDoHouses), + do_valuations: String(bfDoValuations), + dry_run: String(bfDryRun), + }); + return apiFetch( + `/api/v1/admin/scrape/cian-backfill-history?${qs.toString()}`, + { method: "POST" }, + ); + }, + }); + + // ── cian-price-history ───────────────────────────────────────────────── + const [phBatchSize, setPhBatchSize] = useState("50"); + const [phListingId, setPhListingId] = useState(""); + + const priceHistoryMut = useMutation({ + mutationFn: () => + apiFetch("/api/v1/admin/scrape/cian-price-history", { + method: "POST", + body: JSON.stringify({ + batch_size: parseInt(phBatchSize, 10), + listing_id: phListingId !== "" ? parseInt(phListingId, 10) : null, + }), + }), + }); + + // ── cian-full-load ───────────────────────────────────────────────────── + const [flDelay, setFlDelay] = useState("1.5"); + const [flConcurrency, setFlConcurrency] = useState("5"); + const [flPriceCap, setFlPriceCap] = useState("1400"); + const [flResumeRunId, setFlResumeRunId] = useState(""); + + const fullLoadMut = useMutation({ + mutationFn: () => + apiFetch("/api/v1/admin/scrape/cian-full-load", { + method: "POST", + body: JSON.stringify({ + price_cap_per_bucket: parseInt(flPriceCap, 10), + request_delay_sec: parseFloat(flDelay), + concurrency: parseInt(flConcurrency, 10), + enrich_detail: false, + detail_top_n: 0, + resume_run_id: flResumeRunId !== "" ? parseInt(flResumeRunId, 10) : null, + }), + }), + }); + + return ( + <> +
+

5. Backfill — история цен (detail)

+

+ Дозаполняет offer_price_history / houses_price_dynamics для листингов и ЖК без истории. +

+
{ + e.preventDefault(); + backfillHistoryMut.mutate(); + }} + > + + + + + + +
+ +
+ +
+

6. Backfill — price-history (T8a)

+

+ T8a: Дозаполняет offer_price_history через detail-страницы (curl_cffi). Идемпотентен. +

+
{ + e.preventDefault(); + priceHistoryMut.mutate(); + }} + > + + + +
+ +
+ +
+

7. Full Load (полный обход без anchor'ов)

+

+ Exhaustive Cian ЕКБ: адаптивное деление по цене, пагинация каждого бакета. Запускается в background. +

+
{ + e.preventDefault(); + fullLoadMut.mutate(); + }} + > + + + + + +
+ +
); } @@ -1296,6 +1714,221 @@ function YandexAdvancedTriggers() { + + + + ); +} + +// ── Yandex Backfill section ──────────────────────────────────────────────── + +function YandexBackfillSection() { + // ── yandex-address-backfill ──────────────────────────────────────────── + const [abLimit, setAbLimit] = useState("200"); + const [abDelay, setAbDelay] = useState("3.0"); + + const addressBackfillMut = useMutation({ + mutationFn: () => + apiFetch("/api/v1/admin/scrape/yandex-address-backfill", { + method: "POST", + body: JSON.stringify({ + limit: parseInt(abLimit, 10), + request_delay_sec: parseFloat(abDelay), + }), + }), + }); + + // ── yandex-full-load ────────────────────────────────────────────────── + const [flDelay, setFlDelay] = useState("2.0"); + const [flConcurrency, setFlConcurrency] = useState("4"); + const [flPriceCap, setFlPriceCap] = useState("500"); + const [flResumeRunId, setFlResumeRunId] = useState(""); + + const fullLoadMut = useMutation({ + mutationFn: () => + apiFetch("/api/v1/admin/scrape/yandex-full-load", { + method: "POST", + body: JSON.stringify({ + price_cap_per_bucket: parseInt(flPriceCap, 10), + request_delay_sec: parseFloat(flDelay), + concurrency: parseInt(flConcurrency, 10), + resume_run_id: flResumeRunId !== "" ? parseInt(flResumeRunId, 10) : null, + }), + }), + }); + + // ── yandex-newbuilding-sweep ─────────────────────────────────────────── + const [nbLimit, setNbLimit] = useState("5"); + const [nbDelay, setNbDelay] = useState("8.0"); + const [nbCity, setNbCity] = useState("ekaterinburg"); + const [nbForce, setNbForce] = useState(false); + + const nbSweepMut = useMutation({ + mutationFn: () => + apiFetch("/api/v1/admin/scrape/yandex-newbuilding-sweep", { + method: "POST", + body: JSON.stringify({ + limit: parseInt(nbLimit, 10), + request_delay_sec: parseFloat(nbDelay), + force: nbForce, + city: nbCity, + }), + }), + }); + + return ( + <> +
+

5. Backfill — address (T10)

+

+ Дозаполняет listings.address для Yandex-листингов без номера дома (detail-страница → полный адрес). +

+
{ + e.preventDefault(); + addressBackfillMut.mutate(); + }} + > + + + +
+ +
+ +
+

6. Full Load (полный обход без anchor'ов)

+

+ Exhaustive Yandex ЕКБ: адаптивное деление по цене, пагинация каждого бакета. Background. +

+
{ + e.preventDefault(); + fullLoadMut.mutate(); + }} + > + + + + + +
+ +
+ +
+

7. Newbuilding enrichment sweep

+

+ Обогащение pending ЖК-домов: resolve slug → fetch_jk → UPSERT yandex_jk_enrichment. +

+
{ + e.preventDefault(); + nbSweepMut.mutate(); + }} + > + + + + + +
+ +
); } @@ -1378,6 +2011,188 @@ function PerSourceDelayEditor({ source, delayMin, delayMax }: PerSourceDelayEdit ); } +// ── Cross-source Backfill & Sync (System tab) ───────────────────────────── + +function HouseImvBackfillSection() { + const [batchSize, setBatchSize] = useState("50"); + const [delay, setDelay] = useState("5.0"); + const [onlyStatus, setOnlyStatus] = useState("pending"); + const [houseId, setHouseId] = useState(""); + + const imvMut = useMutation({ + mutationFn: () => + apiFetch("/api/v1/admin/scrape/house-imv-backfill", { + method: "POST", + body: JSON.stringify({ + batch_size: parseInt(batchSize, 10), + request_delay_sec: parseFloat(delay), + only_status: onlyStatus, + house_id: houseId !== "" ? parseInt(houseId, 10) : null, + }), + }), + }); + + return ( +
+

House IMV Backfill (T7)

+

+ Batch оценка домов через Avito IMV API. Итерирует дома по{" "} + imv_status, сохраняет в{" "} + house_imv_evaluations / house_placement_history /{" "} + house_suggestions. Не снижать delay ниже 3s. +

+
{ + e.preventDefault(); + imvMut.mutate(); + }} + > + + + + + +
+ +
+ ); +} + +function GeocodeSection() { + const [batchSize, setGcBatchSize] = useState("200"); + const [dryRun, setGcDryRun] = useState(false); + const [background, setGcBackground] = useState(false); + + const statusQ = useQuery({ + queryKey: ["geocode-missing-status"], + queryFn: () => + apiFetch("/api/v1/admin/scrape/geocode-missing-listings/status"), + staleTime: 30_000, + refetchInterval: 60_000, + retry: 1, + }); + + const geocodeMut = useMutation({ + mutationFn: () => { + const qs = new URLSearchParams({ + batch_size: batchSize, + dry_run: String(dryRun), + background: String(background), + }); + return apiFetch( + `/api/v1/admin/scrape/geocode-missing-listings?${qs.toString()}`, + { method: "POST" }, + ); + }, + }); + + return ( +
+

Geocode missing listings

+

+ Дозаполняет координаты листингов через Nominatim (address-dedup). + Безопасно повторять. Background=true → возвращает немедленно, статус — в логах. +

+ + {/* Status block */} + {statusQ.isError && ( +

+ Ошибка загрузки статуса: {statusQ.error.message} +

+ )} + {statusQ.isSuccess && ( +
+
+ Pending listings: + {statusQ.data.total_pending_listings.toLocaleString("ru-RU")} +
+
+ Уникальных адресов: + {statusQ.data.unique_addresses_pending.toLocaleString("ru-RU")} +
+
+ ~Минут (Nominatim): + {statusQ.data.estimated_nominatim_minutes} +
+
+ Geocode cache (active): + {statusQ.data.cache_size_active.toLocaleString("ru-RU")} +
+
+ )} + +
{ + e.preventDefault(); + geocodeMut.mutate(); + }} + > + + + + +
+ +
+ ); +} + // ── Provider configs ─────────────────────────────────────────────────────── interface ProviderConfig { @@ -1502,9 +2317,19 @@ export default function ScrapersUnifiedPage() { {/* Section 1: Система */} - {/* Section 2: Глобально */} + {/* Section 2: Pacing */} + + + {/* Section 3: Глобально */} + {/* Section 4: Backfill & Sync (cross-source) */} + + + + {/* Section 5: Качество данных */} + + {/* Tabs: Avito / Cian / Yandex */}
{/* Tab bar */} diff --git a/tradein-mvp/frontend/src/components/scrapers/DataQualitySection.tsx b/tradein-mvp/frontend/src/components/scrapers/DataQualitySection.tsx new file mode 100644 index 00000000..4ff15539 --- /dev/null +++ b/tradein-mvp/frontend/src/components/scrapers/DataQualitySection.tsx @@ -0,0 +1,224 @@ +"use client"; + +import React from "react"; +import { useQuery } from "@tanstack/react-query"; +import { apiFetch } from "@/lib/api"; + +// ── Types (per contract) ─────────────────────────────────────────────────── + +interface SourceQuality { + source: string; + active_count: number; + fields: Record; +} + +interface HousesEnrichment { + total: number; + validated_pct: number; + rating_pct: number; + house_type_pct: number; + reviews_count: number; +} + +interface DataQualityResp { + sources: SourceQuality[]; + houses: HousesEnrichment; +} + +// ── Hook ────────────────────────────────────────────────────────────────── + +function useDataQuality() { + return useQuery({ + queryKey: ["scraper-data-quality"], + queryFn: () => + apiFetch("/api/v1/admin/scraper/data-quality"), + staleTime: 60_000, + refetchInterval: 120_000, + retry: 1, + }); +} + +// ── Fill cell ───────────────────────────────────────────────────────────── + +function FillCell({ pct }: { pct: number }) { + const isLow = pct < 50; + return ( + + {pct.toFixed(1)}% + + ); +} + +// ── DataQualitySection (exported) ───────────────────────────────────────── + +export function DataQualitySection() { + const qualityQ = useDataQuality(); + + // Collect all unique field names across all sources + const allFields: string[] = qualityQ.isSuccess + ? [ + ...new Set( + qualityQ.data.sources.flatMap((s) => Object.keys(s.fields)), + ), + ].sort() + : []; + + return ( +
+

Качество данных / Coverage

+

+ Процент заполнения ключевых полей по активным листингам и обогащению + домов. Красный — менее 50%. Обновление каждые 2 мин. +

+ + {/* Graceful degradation: API not yet deployed */} + {qualityQ.isError && ( +

+ coverage API недоступен + {" — "} + + {qualityQ.error.message} + +

+ )} + + {qualityQ.isPending && ( +

Загрузка данных coverage…

+ )} + + {qualityQ.isSuccess && ( + <> + {/* Sources × fields table */} + {qualityQ.data.sources.length > 0 && allFields.length > 0 && ( +
+ + + + + + {allFields.map((f) => ( + + ))} + + + + {qualityQ.data.sources.map((row) => ( + + + + {allFields.map((f) => ( + + ))} + + ))} + +
Source + Активных + + {f} +
+ {row.source} + + {row.active_count.toLocaleString("ru-RU")} +
+
+ )} + + {qualityQ.data.sources.length === 0 && ( +

Нет данных по источникам.

+ )} + + {/* Houses enrichment metrics */} +
+

+ Обогащение домов +

+
+
+ Всего домов: + + {qualityQ.data.houses.total.toLocaleString("ru-RU")} + +
+
+ Валидировано: + + {qualityQ.data.houses.validated_pct.toFixed(1)}% + +
+
+ С рейтингом: + + {qualityQ.data.houses.rating_pct.toFixed(1)}% + +
+
+ С типом дома: + + {qualityQ.data.houses.house_type_pct.toFixed(1)}% + +
+
+ Отзывов (sum): + + {qualityQ.data.houses.reviews_count.toLocaleString("ru-RU")} + +
+
+
+ + )} +
+ ); +} diff --git a/tradein-mvp/frontend/src/components/scrapers/PacingControl.tsx b/tradein-mvp/frontend/src/components/scrapers/PacingControl.tsx new file mode 100644 index 00000000..6a1d07d8 --- /dev/null +++ b/tradein-mvp/frontend/src/components/scrapers/PacingControl.tsx @@ -0,0 +1,198 @@ +"use client"; + +import React, { useState, useEffect } from "react"; +import { useQuery, useMutation, useQueryClient } from "@tanstack/react-query"; +import { apiFetch } from "@/lib/api"; + +// ── Types (per contract) ─────────────────────────────────────────────────── + +interface PacingProvider { + source: "avito" | "cian" | "yandex" | "generic"; + interval_s: number; + env_default_s: number; +} + +interface PacingResp { + providers: PacingProvider[]; +} + +interface PacingUpdateResp { + ok: boolean; + source: string; + interval_s: number; +} + +// ── Hooks ────────────────────────────────────────────────────────────────── + +function usePacing() { + return useQuery({ + queryKey: ["scraper-pacing"], + queryFn: () => apiFetch("/api/v1/admin/scraper/pacing"), + staleTime: 30_000, + retry: 1, + }); +} + +function useUpdatePacing(source: string) { + const qc = useQueryClient(); + return useMutation({ + mutationFn: ({ interval_s }) => + apiFetch( + `/api/v1/admin/scraper/pacing/${encodeURIComponent(source)}`, + { + method: "PUT", + body: JSON.stringify({ interval_s }), + }, + ), + onSuccess: () => { + void qc.invalidateQueries({ queryKey: ["scraper-pacing"] }); + }, + }); +} + +// ── Per-provider row ─────────────────────────────────────────────────────── + +interface PacingRowProps { + provider: PacingProvider; +} + +function PacingRow({ provider }: PacingRowProps) { + const [value, setValue] = useState(String(provider.interval_s)); + const updateMut = useUpdatePacing(provider.source); + + // Sync when upstream data refreshes + useEffect(() => { + setValue(String(provider.interval_s)); + }, [provider.interval_s]); + + const numValue = parseFloat(value); + const isValid = !isNaN(numValue) && numValue >= 0 && numValue <= 60; + + return ( + + + {provider.source} + + +
+ setValue(e.target.value)} + aria-label={`Pacing interval for ${provider.source} (seconds)`} + style={{ + width: 72, + padding: "3px 6px", + fontSize: "0.85rem", + border: "1px solid var(--border-card, #e6e8ec)", + borderRadius: 6, + }} + /> + setValue(e.target.value)} + aria-label={`Pacing slider for ${provider.source}`} + style={{ width: 120 }} + /> + + с (по умолч. {provider.env_default_s} с — сбросится при рестарте) + +
+ + + + {updateMut.isSuccess && updateMut.data.ok && ( + + {updateMut.data.interval_s} с — ок + + )} + {updateMut.isSuccess && !updateMut.data.ok && ( + + не применено + + )} + {updateMut.isError && ( + + {updateMut.error.message} + + )} + + + ); +} + +// ── PacingSection (exported) ─────────────────────────────────────────────── + +export function PacingSection() { + const pacingQ = usePacing(); + + return ( +
+

Pacing (интервалы запросов)

+

+ Минимальный интервал между запросами скраппера для каждого провайдера. + 0 = без искусственной паузы. Изменение вступает в силу немедленно; + сбрасывается при рестарте контейнера (постоянное значение — через ENV). +

+ + {/* Graceful degradation: API not yet deployed */} + {pacingQ.isError && ( +

+ pacing API недоступен + {" — "} + + {pacingQ.error.message} + +

+ )} + + {pacingQ.isPending && ( +

Загрузка pacing…

+ )} + + {pacingQ.isSuccess && pacingQ.data.providers.length === 0 && ( +

Нет данных pacing от API.

+ )} + + {pacingQ.isSuccess && pacingQ.data.providers.length > 0 && ( + + + + + + + + + + {pacingQ.data.providers.map((p) => ( + + ))} + +
ПровайдерИнтервал (0–60 с)Действие
+ )} +
+ ); +} diff --git a/tradein-mvp/frontend/src/components/scrapers/RunsTable.tsx b/tradein-mvp/frontend/src/components/scrapers/RunsTable.tsx index 45d89acc..a6549a3f 100644 --- a/tradein-mvp/frontend/src/components/scrapers/RunsTable.tsx +++ b/tradein-mvp/frontend/src/components/scrapers/RunsTable.tsx @@ -37,15 +37,35 @@ interface RunsListResp { const RUN_STATUS_ALL = ["", "running", "done", "failed", "cancelled", "zombie", "banned"] as const; type RunStatusFilter = (typeof RUN_STATUS_ALL)[number]; +// "" means "all sources"; otherwise a specific source prefix (avito / cian / yandex) +const RUN_SOURCE_FILTERS = ["", "avito", "cian", "yandex"] as const; +type RunSourceFilter = (typeof RUN_SOURCE_FILTERS)[number]; + +const SOURCE_FILTER_LABELS: Record = { + "": "Все", + avito: "Avito", + cian: "Cian", + yandex: "Yandex", +}; + function useScraperRuns( source: ScraperSource, status: RunStatusFilter, + sourceFilter: RunSourceFilter, limit = 20, ) { return useQuery({ - queryKey: ["scrape-runs", source, status, limit], + queryKey: ["scrape-runs", source, status, sourceFilter, limit], queryFn: () => { - const qs = new URLSearchParams({ source, limit: String(limit) }); + const qs = new URLSearchParams({ limit: String(limit) }); + // When a specific sourceFilter is chosen, ignore the tab-level source + // and pass it verbatim as the ?source= param + if (sourceFilter) { + qs.set("source", sourceFilter); + } else { + // fallback: filter by the current tab provider + qs.set("source", source); + } if (status) qs.set("status", status); return apiFetch( `/api/v1/admin/scrape/runs?${qs.toString()}`, @@ -154,17 +174,17 @@ interface RunsTableProps { } export function RunsTable({ source }: RunsTableProps) { - const [statusFilter, setStatusFilter] = - useState(""); + const [statusFilter, setStatusFilter] = useState(""); + const [sourceFilter, setSourceFilter] = useState(""); const qc = useQueryClient(); - const runsQ = useScraperRuns(source, statusFilter); + const runsQ = useScraperRuns(source, statusFilter, sourceFilter); const cancelMut = useCancelCitySweep(source); function handleCancel(runId: number) { cancelMut.mutate(runId, { onSuccess: () => { void qc.invalidateQueries({ - queryKey: ["scrape-runs", source, statusFilter], + queryKey: ["scrape-runs", source, statusFilter, sourceFilter], }); }, }); @@ -177,6 +197,37 @@ export function RunsTable({ source }: RunsTableProps) { Последние 20 прогонов. Автообновление каждые 8 сек.

+ {/* Source filter */} +
+ + +
+ {/* Status filter */}
{RUN_STATUS_ALL.map((s) => (