gendesign/tradein-mvp/frontend/src/components/scrapers/DataQualitySection.tsx
bot-backend 00cd30b765
All checks were successful
CI / changes (pull_request) Successful in 6s
CI / backend-tests (pull_request) Has been skipped
CI / frontend-tests (pull_request) Has been skipped
CI / openapi-codegen-check (pull_request) Has been skipped
feat(scrapers/ui): backfill+full-load триггеры, source-фильтр runs, pacing-слайдер, coverage-таблица
Часть 1 — недостающие триггеры:
- CianBackfillSection: cian-backfill-history, cian-price-history (T8a), cian-full-load
- YandexBackfillSection: yandex-address-backfill (T10), yandex-full-load, yandex-newbuilding-sweep
- CianAutoLoginButton: POST /scrape/cian/auto-login в секцию Cookies
- HouseImvBackfillSection: house-imv-backfill (T7) в системный раздел
- GeocodeSection: geocode-missing-listings trigger + status polling

Часть 2 — source-фильтр в RunsTable:
- dropdown «Все / Avito / Cian / Yandex» + aria-label
- фильтр прокидывается в queryKey и query-param ?source=

Часть 3 — PacingControl:
- GET /api/v1/admin/scraper/pacing → PacingSection
- number-input + range slider (0–60с, шаг 0.5) + PUT /pacing/{source}
- env_default_s подсказка; graceful-degradation на ошибку/404

Часть 4 — DataQualitySection:
- GET /api/v1/admin/scraper/data-quality → таблица source × поле (% fill)
- <50% → красный акцент; th scope="col"/"row" a11y
- блок houses enrichment метрик; graceful-degradation на ошибку/404
2026-06-20 12:50:04 +03:00

224 lines
7.7 KiB
TypeScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"use client";
import React from "react";
import { useQuery } from "@tanstack/react-query";
import { apiFetch } from "@/lib/api";
// ── Types (per contract) ───────────────────────────────────────────────────
interface SourceQuality {
source: string;
active_count: number;
fields: Record<string, number>;
}
interface HousesEnrichment {
total: number;
validated_pct: number;
rating_pct: number;
house_type_pct: number;
reviews_count: number;
}
interface DataQualityResp {
sources: SourceQuality[];
houses: HousesEnrichment;
}
// ── Hook ──────────────────────────────────────────────────────────────────
function useDataQuality() {
return useQuery<DataQualityResp>({
queryKey: ["scraper-data-quality"],
queryFn: () =>
apiFetch<DataQualityResp>("/api/v1/admin/scraper/data-quality"),
staleTime: 60_000,
refetchInterval: 120_000,
retry: 1,
});
}
// ── Fill cell ─────────────────────────────────────────────────────────────
function FillCell({ pct }: { pct: number }) {
const isLow = pct < 50;
return (
<td
style={{
textAlign: "right",
fontSize: "0.8rem",
fontWeight: isLow ? 600 : 400,
color: isLow ? "var(--danger, #b3261e)" : "var(--fg-primary, #1f2937)",
background: isLow ? "var(--danger-soft, #fef2f2)" : undefined,
}}
>
{pct.toFixed(1)}%
</td>
);
}
// ── DataQualitySection (exported) ─────────────────────────────────────────
export function DataQualitySection() {
const qualityQ = useDataQuality();
// Collect all unique field names across all sources
const allFields: string[] = qualityQ.isSuccess
? [
...new Set<string>(
qualityQ.data.sources.flatMap((s) => Object.keys(s.fields)),
),
].sort()
: [];
return (
<section className="scraper-section">
<h2>Качество данных / Coverage</h2>
<p className="scraper-hint">
Процент заполнения ключевых полей по активным листингам и обогащению
домов. Красный менее 50%. Обновление каждые 2 мин.
</p>
{/* Graceful degradation: API not yet deployed */}
{qualityQ.isError && (
<p
className="scraper-result scraper-result--error"
style={{ marginBottom: 0 }}
>
coverage API недоступен
{" — "}
<span style={{ color: "var(--fg-secondary, #5b6066)" }}>
{qualityQ.error.message}
</span>
</p>
)}
{qualityQ.isPending && (
<p className="scraper-hint">Загрузка данных coverage</p>
)}
{qualityQ.isSuccess && (
<>
{/* Sources × fields table */}
{qualityQ.data.sources.length > 0 && allFields.length > 0 && (
<div style={{ overflowX: "auto", marginBottom: 20 }}>
<table className="runs-table">
<thead>
<tr>
<th scope="col">Source</th>
<th scope="col" style={{ textAlign: "right" }}>
Активных
</th>
{allFields.map((f) => (
<th
key={f}
scope="col"
style={{ textAlign: "right", whiteSpace: "nowrap" }}
>
{f}
</th>
))}
</tr>
</thead>
<tbody>
{qualityQ.data.sources.map((row) => (
<tr key={row.source}>
<th scope="row">
<code style={{ fontWeight: 600 }}>{row.source}</code>
</th>
<td style={{ textAlign: "right", fontSize: "0.85rem" }}>
{row.active_count.toLocaleString("ru-RU")}
</td>
{allFields.map((f) => (
<FillCell
key={f}
pct={row.fields[f] !== undefined ? row.fields[f] : 0}
/>
))}
</tr>
))}
</tbody>
</table>
</div>
)}
{qualityQ.data.sources.length === 0 && (
<p className="scraper-hint">Нет данных по источникам.</p>
)}
{/* Houses enrichment metrics */}
<div>
<p
style={{
fontSize: "0.85rem",
fontWeight: 600,
marginBottom: 8,
}}
>
Обогащение домов
</p>
<div className="schedule-status">
<div className="schedule-status__row">
<span className="schedule-status__label">Всего домов:</span>
<span>
{qualityQ.data.houses.total.toLocaleString("ru-RU")}
</span>
</div>
<div className="schedule-status__row">
<span className="schedule-status__label">Валидировано:</span>
<span
style={{
color:
qualityQ.data.houses.validated_pct < 50
? "var(--danger, #b3261e)"
: undefined,
fontWeight:
qualityQ.data.houses.validated_pct < 50 ? 600 : 400,
}}
>
{qualityQ.data.houses.validated_pct.toFixed(1)}%
</span>
</div>
<div className="schedule-status__row">
<span className="schedule-status__label">С рейтингом:</span>
<span
style={{
color:
qualityQ.data.houses.rating_pct < 50
? "var(--danger, #b3261e)"
: undefined,
fontWeight:
qualityQ.data.houses.rating_pct < 50 ? 600 : 400,
}}
>
{qualityQ.data.houses.rating_pct.toFixed(1)}%
</span>
</div>
<div className="schedule-status__row">
<span className="schedule-status__label">С типом дома:</span>
<span
style={{
color:
qualityQ.data.houses.house_type_pct < 50
? "var(--danger, #b3261e)"
: undefined,
fontWeight:
qualityQ.data.houses.house_type_pct < 50 ? 600 : 400,
}}
>
{qualityQ.data.houses.house_type_pct.toFixed(1)}%
</span>
</div>
<div className="schedule-status__row">
<span className="schedule-status__label">Отзывов (sum):</span>
<span>
{qualityQ.data.houses.reviews_count.toLocaleString("ru-RU")}
</span>
</div>
</div>
</div>
</>
)}
</section>
);
}