Часть 1 — недостающие триггеры:
- CianBackfillSection: cian-backfill-history, cian-price-history (T8a), cian-full-load
- YandexBackfillSection: yandex-address-backfill (T10), yandex-full-load, yandex-newbuilding-sweep
- CianAutoLoginButton: POST /scrape/cian/auto-login в секцию Cookies
- HouseImvBackfillSection: house-imv-backfill (T7) в системный раздел
- GeocodeSection: geocode-missing-listings trigger + status polling
Часть 2 — source-фильтр в RunsTable:
- dropdown «Все / Avito / Cian / Yandex» + aria-label
- фильтр прокидывается в queryKey и query-param ?source=
Часть 3 — PacingControl:
- GET /api/v1/admin/scraper/pacing → PacingSection
- number-input + range slider (0–60с, шаг 0.5) + PUT /pacing/{source}
- env_default_s подсказка; graceful-degradation на ошибку/404
Часть 4 — DataQualitySection:
- GET /api/v1/admin/scraper/data-quality → таблица source × поле (% fill)
- <50% → красный акцент; th scope="col"/"row" a11y
- блок houses enrichment метрик; graceful-degradation на ошибку/404
224 lines
7.7 KiB
TypeScript
224 lines
7.7 KiB
TypeScript
"use client";
|
||
|
||
import React from "react";
|
||
import { useQuery } from "@tanstack/react-query";
|
||
import { apiFetch } from "@/lib/api";
|
||
|
||
// ── Types (per contract) ───────────────────────────────────────────────────
|
||
|
||
interface SourceQuality {
|
||
source: string;
|
||
active_count: number;
|
||
fields: Record<string, number>;
|
||
}
|
||
|
||
interface HousesEnrichment {
|
||
total: number;
|
||
validated_pct: number;
|
||
rating_pct: number;
|
||
house_type_pct: number;
|
||
reviews_count: number;
|
||
}
|
||
|
||
interface DataQualityResp {
|
||
sources: SourceQuality[];
|
||
houses: HousesEnrichment;
|
||
}
|
||
|
||
// ── Hook ──────────────────────────────────────────────────────────────────
|
||
|
||
function useDataQuality() {
|
||
return useQuery<DataQualityResp>({
|
||
queryKey: ["scraper-data-quality"],
|
||
queryFn: () =>
|
||
apiFetch<DataQualityResp>("/api/v1/admin/scraper/data-quality"),
|
||
staleTime: 60_000,
|
||
refetchInterval: 120_000,
|
||
retry: 1,
|
||
});
|
||
}
|
||
|
||
// ── Fill cell ─────────────────────────────────────────────────────────────
|
||
|
||
function FillCell({ pct }: { pct: number }) {
|
||
const isLow = pct < 50;
|
||
return (
|
||
<td
|
||
style={{
|
||
textAlign: "right",
|
||
fontSize: "0.8rem",
|
||
fontWeight: isLow ? 600 : 400,
|
||
color: isLow ? "var(--danger, #b3261e)" : "var(--fg-primary, #1f2937)",
|
||
background: isLow ? "var(--danger-soft, #fef2f2)" : undefined,
|
||
}}
|
||
>
|
||
{pct.toFixed(1)}%
|
||
</td>
|
||
);
|
||
}
|
||
|
||
// ── DataQualitySection (exported) ─────────────────────────────────────────
|
||
|
||
export function DataQualitySection() {
|
||
const qualityQ = useDataQuality();
|
||
|
||
// Collect all unique field names across all sources
|
||
const allFields: string[] = qualityQ.isSuccess
|
||
? [
|
||
...new Set<string>(
|
||
qualityQ.data.sources.flatMap((s) => Object.keys(s.fields)),
|
||
),
|
||
].sort()
|
||
: [];
|
||
|
||
return (
|
||
<section className="scraper-section">
|
||
<h2>Качество данных / Coverage</h2>
|
||
<p className="scraper-hint">
|
||
Процент заполнения ключевых полей по активным листингам и обогащению
|
||
домов. Красный — менее 50%. Обновление каждые 2 мин.
|
||
</p>
|
||
|
||
{/* Graceful degradation: API not yet deployed */}
|
||
{qualityQ.isError && (
|
||
<p
|
||
className="scraper-result scraper-result--error"
|
||
style={{ marginBottom: 0 }}
|
||
>
|
||
coverage API недоступен
|
||
{" — "}
|
||
<span style={{ color: "var(--fg-secondary, #5b6066)" }}>
|
||
{qualityQ.error.message}
|
||
</span>
|
||
</p>
|
||
)}
|
||
|
||
{qualityQ.isPending && (
|
||
<p className="scraper-hint">Загрузка данных coverage…</p>
|
||
)}
|
||
|
||
{qualityQ.isSuccess && (
|
||
<>
|
||
{/* Sources × fields table */}
|
||
{qualityQ.data.sources.length > 0 && allFields.length > 0 && (
|
||
<div style={{ overflowX: "auto", marginBottom: 20 }}>
|
||
<table className="runs-table">
|
||
<thead>
|
||
<tr>
|
||
<th scope="col">Source</th>
|
||
<th scope="col" style={{ textAlign: "right" }}>
|
||
Активных
|
||
</th>
|
||
{allFields.map((f) => (
|
||
<th
|
||
key={f}
|
||
scope="col"
|
||
style={{ textAlign: "right", whiteSpace: "nowrap" }}
|
||
>
|
||
{f}
|
||
</th>
|
||
))}
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
{qualityQ.data.sources.map((row) => (
|
||
<tr key={row.source}>
|
||
<th scope="row">
|
||
<code style={{ fontWeight: 600 }}>{row.source}</code>
|
||
</th>
|
||
<td style={{ textAlign: "right", fontSize: "0.85rem" }}>
|
||
{row.active_count.toLocaleString("ru-RU")}
|
||
</td>
|
||
{allFields.map((f) => (
|
||
<FillCell
|
||
key={f}
|
||
pct={row.fields[f] !== undefined ? row.fields[f] : 0}
|
||
/>
|
||
))}
|
||
</tr>
|
||
))}
|
||
</tbody>
|
||
</table>
|
||
</div>
|
||
)}
|
||
|
||
{qualityQ.data.sources.length === 0 && (
|
||
<p className="scraper-hint">Нет данных по источникам.</p>
|
||
)}
|
||
|
||
{/* Houses enrichment metrics */}
|
||
<div>
|
||
<p
|
||
style={{
|
||
fontSize: "0.85rem",
|
||
fontWeight: 600,
|
||
marginBottom: 8,
|
||
}}
|
||
>
|
||
Обогащение домов
|
||
</p>
|
||
<div className="schedule-status">
|
||
<div className="schedule-status__row">
|
||
<span className="schedule-status__label">Всего домов:</span>
|
||
<span>
|
||
{qualityQ.data.houses.total.toLocaleString("ru-RU")}
|
||
</span>
|
||
</div>
|
||
<div className="schedule-status__row">
|
||
<span className="schedule-status__label">Валидировано:</span>
|
||
<span
|
||
style={{
|
||
color:
|
||
qualityQ.data.houses.validated_pct < 50
|
||
? "var(--danger, #b3261e)"
|
||
: undefined,
|
||
fontWeight:
|
||
qualityQ.data.houses.validated_pct < 50 ? 600 : 400,
|
||
}}
|
||
>
|
||
{qualityQ.data.houses.validated_pct.toFixed(1)}%
|
||
</span>
|
||
</div>
|
||
<div className="schedule-status__row">
|
||
<span className="schedule-status__label">С рейтингом:</span>
|
||
<span
|
||
style={{
|
||
color:
|
||
qualityQ.data.houses.rating_pct < 50
|
||
? "var(--danger, #b3261e)"
|
||
: undefined,
|
||
fontWeight:
|
||
qualityQ.data.houses.rating_pct < 50 ? 600 : 400,
|
||
}}
|
||
>
|
||
{qualityQ.data.houses.rating_pct.toFixed(1)}%
|
||
</span>
|
||
</div>
|
||
<div className="schedule-status__row">
|
||
<span className="schedule-status__label">С типом дома:</span>
|
||
<span
|
||
style={{
|
||
color:
|
||
qualityQ.data.houses.house_type_pct < 50
|
||
? "var(--danger, #b3261e)"
|
||
: undefined,
|
||
fontWeight:
|
||
qualityQ.data.houses.house_type_pct < 50 ? 600 : 400,
|
||
}}
|
||
>
|
||
{qualityQ.data.houses.house_type_pct.toFixed(1)}%
|
||
</span>
|
||
</div>
|
||
<div className="schedule-status__row">
|
||
<span className="schedule-status__label">Отзывов (sum):</span>
|
||
<span>
|
||
{qualityQ.data.houses.reviews_count.toLocaleString("ru-RU")}
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
</>
|
||
)}
|
||
</section>
|
||
);
|
||
}
|