From af3da0f02e77a13f0f091427e8a712867d0d0f8c Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 23 May 2026 17:01:12 +0300 Subject: [PATCH] =?UTF-8?q?feat(tradein):=20Stage=204c=20=E2=80=94=20scrap?= =?UTF-8?q?ers=20admin=20pages=20(Avito=20real=20+=20Cian/Yandex=20stubs)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - NEW Topbar component (shared header с 6 nav items: Оценка/История/Кэш/Авито/Циан/Яндекс) - NEW /scrapers/avito — 4 trigger forms по pattern frontend/admin/scrape/objective: * Search around point (lat/lon/radius) → POST /admin/scrape * Houses Catalog enrichment (house_url) → POST /admin/scrape/avito-house * Detail page enrichment (item_url) → POST /admin/scrape/avito-detail * IMV evaluation debug (9 params) → POST /admin/scrape/avito-imv TanStack useMutation; JSON response panel; loading/error states - NEW /scrapers/cian + /scrapers/yandex — stub pages с pointer на cron/curl - UPDATE page.tsx — replace inline topbar with - CSS: .scraper-page, .scraper-section, .scraper-form, .scraper-result Бэкенд endpoints из PR #460 (Stage 4a). tsc + next build pass. --- tradein-mvp/frontend/src/app/page.tsx | 19 +- .../frontend/src/app/scrapers/avito/page.tsx | 417 ++++++++++++++++++ .../frontend/src/app/scrapers/cian/page.tsx | 24 + .../frontend/src/app/scrapers/yandex/page.tsx | 24 + .../src/components/trade-in/Topbar.tsx | 44 ++ .../src/components/trade-in/trade-in.css | 133 ++++++ 6 files changed, 644 insertions(+), 17 deletions(-) create mode 100644 tradein-mvp/frontend/src/app/scrapers/avito/page.tsx create mode 100644 tradein-mvp/frontend/src/app/scrapers/cian/page.tsx create mode 100644 tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx create mode 100644 tradein-mvp/frontend/src/components/trade-in/Topbar.tsx diff --git a/tradein-mvp/frontend/src/app/page.tsx b/tradein-mvp/frontend/src/app/page.tsx index 70566f91..aacfbdd1 100644 --- a/tradein-mvp/frontend/src/app/page.tsx +++ b/tradein-mvp/frontend/src/app/page.tsx @@ -11,8 +11,8 @@ import { useRouter } from "next/navigation"; import "@/components/trade-in/trade-in.css"; import type { AggregatedEstimate, TradeInEstimateInput } from "@/types/trade-in"; import { useEstimateMutation, useEstimate } from "@/lib/trade-in-api"; -import { API_BASE_URL } from "@/lib/api"; import { EstimateForm } from "@/components/trade-in/EstimateForm"; +import { Topbar } from "@/components/trade-in/Topbar"; import { SourcesProgress } from "@/components/trade-in/SourcesProgress"; import { HeroSummary } from "@/components/trade-in/HeroSummary"; import { IMVBenchmark } from "@/components/trade-in/IMVBenchmark"; @@ -79,22 +79,7 @@ export default function TradeInPage() { return ( <> - {/* Topbar — белый-лейбл бренд */} -
-
-
- TI - PRINZIP - - Trade-In -
- -
-
+
diff --git a/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx b/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx new file mode 100644 index 00000000..23263f2f --- /dev/null +++ b/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx @@ -0,0 +1,417 @@ +"use client"; + +import { useState } from "react"; +import { useMutation } from "@tanstack/react-query"; + +import "@/components/trade-in/trade-in.css"; +import { Topbar } from "@/components/trade-in/Topbar"; +import { apiFetch } from "@/lib/api"; + +// ── Types ────────────────────────────────────────────────────────────────── + +interface ScrapeAroundInput { + lat: number; + lon: number; + radius_m: number; +} + +interface ScrapeAroundResp { + total_fetched: number; + total_inserted: number; + total_updated: number; + by_source: Array<{ + source: string; + fetched: number; + inserted: number; + updated: number; + }>; +} + +interface HouseTriggerResp { + ok: boolean; + house_url: string; + counters: Record; +} + +interface DetailTriggerResp { + ok: boolean; + item_id: string; + updated: boolean; +} + +interface ImvInput { + address: string; + rooms: number; + area_m2: number; + floor: number; + floor_at_home: number; + house_type: string; + renovation_type: string; + has_balcony: boolean; + has_loggia: boolean; +} + +interface ImvResp { + ok: boolean; + cache_key: string; + recommended_price: number; + range: [number, number]; + market_count: number | null; + evaluation_id: number; + history_saved: number; +} + +// ── Mutations ────────────────────────────────────────────────────────────── + +function useScrapeAround() { + return useMutation({ + mutationFn: (input) => + apiFetch("/api/v1/admin/scrape", { + method: "POST", + body: JSON.stringify({ ...input, sources: ["avito"] }), + }), + }); +} + +function useScrapeHouse() { + return useMutation({ + mutationFn: ({ house_url }) => + apiFetch( + `/api/v1/admin/scrape/avito-house?house_url=${encodeURIComponent(house_url)}`, + { method: "POST" }, + ), + }); +} + +function useScrapeDetail() { + return useMutation({ + mutationFn: ({ item_url }) => + apiFetch( + `/api/v1/admin/scrape/avito-detail?item_url=${encodeURIComponent(item_url)}`, + { method: "POST" }, + ), + }); +} + +function useScrapeImv() { + return useMutation({ + mutationFn: (input) => { + const qs = new URLSearchParams({ + address: input.address, + rooms: String(input.rooms), + area_m2: String(input.area_m2), + floor: String(input.floor), + floor_at_home: String(input.floor_at_home), + house_type: input.house_type, + renovation_type: input.renovation_type, + has_balcony: String(input.has_balcony), + has_loggia: String(input.has_loggia), + }); + return apiFetch( + `/api/v1/admin/scrape/avito-imv?${qs.toString()}`, + { method: "POST" }, + ); + }, + }); +} + +// ── Result panel ─────────────────────────────────────────────────────────── + +interface ResultPanelProps { + mut: { + data?: TData; + error: Error | null; + isPending: boolean; + isSuccess: boolean; + }; +} + +function ResultPanel({ mut }: ResultPanelProps) { + if (mut.error) { + return ( +
+ Ошибка: {mut.error.message} +
+ ); + } + if (mut.isSuccess && mut.data) { + return ( +
+
{JSON.stringify(mut.data, null, 2)}
+
+ ); + } + return null; +} + +// ── Page ─────────────────────────────────────────────────────────────────── + +export default function AvitoScraperPage() { + // 1. Around-point + const [lat, setLat] = useState("56.8400"); + const [lon, setLon] = useState("60.6050"); + const [radius, setRadius] = useState("1500"); + const aroundMut = useScrapeAround(); + + // 2. Houses Catalog + const [houseUrl, setHouseUrl] = useState( + "/catalog/houses/ekaterinburg/ul_akademika_postovskogo_17a/3171365", + ); + const houseMut = useScrapeHouse(); + + // 3. Detail + const [itemUrl, setItemUrl] = useState( + "/ekaterinburg/kvartiry/3-k._kvartira_75_m_2026_et._7986882804", + ); + const detailMut = useScrapeDetail(); + + // 4. IMV + const [imvAddress, setImvAddress] = useState( + "Свердловская обл., Екатеринбург, ул. Чайковского, 66А", + ); + const [imvRooms, setImvRooms] = useState("2"); + const [imvArea, setImvArea] = useState("42"); + const [imvFloor, setImvFloor] = useState("4"); + const [imvTotalFloors, setImvTotalFloors] = useState("5"); + const [imvHouseType, setImvHouseType] = useState("panel"); + const [imvRenovation, setImvRenovation] = useState("cosmetic"); + const [imvBalcony, setImvBalcony] = useState(true); + const [imvLoggia, setImvLoggia] = useState(false); + const imvMut = useScrapeImv(); + + return ( + <> + +
+

Avito скрапер — admin триггер

+

+ Ручной запуск sub-pipeline'ов AvitoScraper v2. Для бесплатной + разработки и теста после изменений. +

+ + {/* 1. Search around point */} +
+

1. Search around (общий /admin/scrape)

+

+ Запускает классический cron-pipeline для одного якоря: search → + save_listings. Использовать как baseline. +

+
{ + e.preventDefault(); + aroundMut.mutate({ + lat: parseFloat(lat), + lon: parseFloat(lon), + radius_m: parseInt(radius, 10), + }); + }} + > + + + + +
+ +
+ + {/* 2. Houses Catalog */} +
+

2. Houses Catalog enrichment

+

+ Парсит /catalog/houses/<slug>/<id> — характеристики + дома, отзывы, history. +

+
{ + e.preventDefault(); + houseMut.mutate({ house_url: houseUrl }); + }} + > + + +
+ +
+ + {/* 3. Detail */} +
+

3. Detail page enrichment

+

+ Парсит /ekaterinburg/kvartiry/<...> — 30+ полей + (kitchen_area, owners_count, etc). UPDATE listings WHERE source_id + (listing должен уже существовать). +

+
{ + e.preventDefault(); + detailMut.mutate({ item_url: itemUrl }); + }} + > + + +
+ +
+ + {/* 4. IMV evaluation */} +
+

4. IMV evaluation (debug, без cache)

+

+ Прямой вызов Avito IMV API (2 HTTP requests). Production вызов из + estimator on-demand с 24h cache; здесь — для debug. +

+
{ + e.preventDefault(); + imvMut.mutate({ + address: imvAddress, + rooms: parseInt(imvRooms, 10), + area_m2: parseFloat(imvArea), + floor: parseInt(imvFloor, 10), + floor_at_home: parseInt(imvTotalFloors, 10), + house_type: imvHouseType, + renovation_type: imvRenovation, + has_balcony: imvBalcony, + has_loggia: imvLoggia, + }); + }} + > + + + + + + + + + + +
+ +
+
+ + ); +} diff --git a/tradein-mvp/frontend/src/app/scrapers/cian/page.tsx b/tradein-mvp/frontend/src/app/scrapers/cian/page.tsx new file mode 100644 index 00000000..e239e18a --- /dev/null +++ b/tradein-mvp/frontend/src/app/scrapers/cian/page.tsx @@ -0,0 +1,24 @@ +"use client"; + +import "@/components/trade-in/trade-in.css"; +import { Topbar } from "@/components/trade-in/Topbar"; + +export default function CianScraperPage() { + return ( + <> + +
+

Cian скрапер

+

В разработке.

+
+

+ Cian SERP refactor merged (PR #450), но admin UI ещё не подключён. + Используй cron-scrape.sh или прямой curl к{" "} + POST /api/v1/admin/scrape с{" "} + sources: ["cian"]. +

+
+
+ + ); +} diff --git a/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx b/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx new file mode 100644 index 00000000..47506196 --- /dev/null +++ b/tradein-mvp/frontend/src/app/scrapers/yandex/page.tsx @@ -0,0 +1,24 @@ +"use client"; + +import "@/components/trade-in/trade-in.css"; +import { Topbar } from "@/components/trade-in/Topbar"; + +export default function YandexScraperPage() { + return ( + <> + +
+

Yandex скрапер

+

В разработке.

+
+

+ Yandex SERP refactor merged (PR #462), но admin UI ещё не подключён. + Используй cron-scrape.sh или прямой curl к{" "} + POST /api/v1/admin/scrape с{" "} + sources: ["yandex"]. +

+
+
+ + ); +} diff --git a/tradein-mvp/frontend/src/components/trade-in/Topbar.tsx b/tradein-mvp/frontend/src/components/trade-in/Topbar.tsx new file mode 100644 index 00000000..4f8a9a9c --- /dev/null +++ b/tradein-mvp/frontend/src/components/trade-in/Topbar.tsx @@ -0,0 +1,44 @@ +"use client"; + +import { API_BASE_URL } from "@/lib/api"; + +type ActiveTab = "estimate" | "history" | "cache" | "avito" | "cian" | "yandex"; + +interface TopbarProps { + active: ActiveTab; +} + +const NAV_ITEMS: Array<{ key: ActiveTab; href: string; label: string }> = [ + { key: "estimate", href: "/", label: "Оценка" }, + { key: "history", href: "/history", label: "История" }, + { key: "cache", href: "/cache", label: "Кэш" }, + { key: "avito", href: "/scrapers/avito", label: "Авито" }, + { key: "cian", href: "/scrapers/cian", label: "Циан" }, + { key: "yandex", href: "/scrapers/yandex", label: "Яндекс" }, +]; + +export function Topbar({ active }: TopbarProps) { + return ( +
+
+
+ TI + PRINZIP + + Trade-In +
+ +
+
+ ); +} diff --git a/tradein-mvp/frontend/src/components/trade-in/trade-in.css b/tradein-mvp/frontend/src/components/trade-in/trade-in.css index cb1092fb..c0c6514d 100644 --- a/tradein-mvp/frontend/src/components/trade-in/trade-in.css +++ b/tradein-mvp/frontend/src/components/trade-in/trade-in.css @@ -1239,3 +1239,136 @@ color: #6b7280; font-style: italic; } + +/* ── Stage 4c: scrapers admin pages ── */ + +.scraper-page { + max-width: 960px; + margin: 0 auto; + padding: 24px 20px 60px; +} + +.scraper-h1 { + font-size: 22px; + font-weight: 600; + margin: 0 0 4px; + color: var(--fg-primary, #111111); +} + +.scraper-subtitle { + font-size: 14px; + color: var(--fg-secondary, #5b6066); + margin: 0 0 24px; +} + +.scraper-section { + margin-bottom: 24px; + padding: 20px; + background: var(--bg-card, #ffffff); + border: 1px solid var(--border-card, #e6e8ec); + border-radius: 12px; +} + +.scraper-section h2 { + font-size: 16px; + font-weight: 600; + margin: 0 0 8px; + color: var(--fg-primary, #111111); +} + +.scraper-hint { + font-size: 13px; + color: var(--fg-secondary, #5b6066); + margin: 0 0 16px; +} + +.scraper-form { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(140px, 1fr)); + gap: 12px; + align-items: end; +} + +.scraper-form label { + display: flex; + flex-direction: column; + font-size: 12px; + color: var(--fg-secondary, #5b6066); + gap: 4px; +} + +.scraper-form label.full { + grid-column: 1 / -1; +} + +.scraper-form label.checkbox { + flex-direction: row; + align-items: center; + gap: 6px; + font-size: 13px; + color: var(--fg-primary, #111111); +} + +.scraper-form input, +.scraper-form select { + padding: 8px 10px; + border: 1px solid var(--border-strong, #d1d5db); + border-radius: 6px; + font-size: 14px; + background: #fff; + color: var(--fg-primary, #111111); +} + +.scraper-form input:focus, +.scraper-form select:focus { + outline: 2px solid var(--accent, #1d4ed8); + outline-offset: -1px; + border-color: var(--accent, #1d4ed8); +} + +.scraper-form button { + padding: 8px 16px; + background: var(--accent, #1d4ed8); + color: #fff; + border: none; + border-radius: 6px; + font-size: 14px; + font-weight: 500; + cursor: pointer; + grid-column: 1 / -1; + justify-self: start; +} + +.scraper-form button:hover:not(:disabled) { + background: var(--accent-hover, #1e40af); +} + +.scraper-form button:disabled { + background: var(--border-strong, #d1d5db); + cursor: not-allowed; +} + +.scraper-result { + margin-top: 16px; + padding: 12px; + background: var(--bg-card-alt, #fafbfc); + border: 1px solid var(--border-soft, #eef0f3); + border-radius: 8px; + font-size: 12px; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + max-height: 320px; + overflow: auto; +} + +.scraper-result--error { + background: var(--danger-soft, #fee2e2); + border-color: var(--danger, #b3261e); + color: var(--danger, #b3261e); + font-family: inherit; +} + +.scraper-result pre { + margin: 0; + white-space: pre-wrap; + word-break: break-all; +} -- 2.45.3