gendesign/tradein-mvp/frontend/src/app/scrapers/avito/page.tsx
bot-frontend ee8f72da39
All checks were successful
Deploy Trade-In / changes (push) Successful in 5s
Deploy Trade-In / test (push) Has been skipped
Deploy Trade-In / build-backend (push) Has been skipped
Deploy Trade-In / build-frontend (push) Successful in 1m37s
Deploy Trade-In / deploy (push) Successful in 36s
feat(tradein): unify scraper admin pages (avito/cian/yandex) + /scrapers hub (#878)
Co-authored-by: bot-frontend <bot-frontend@gendsgn.local>
Co-committed-by: bot-frontend <bot-frontend@gendsgn.local>
2026-05-31 09:31:47 +00:00

415 lines
12 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"use client";
import { useState } from "react";
import { useMutation } from "@tanstack/react-query";
import { apiFetch } from "@/lib/api";
import {
ResultPanel,
ScraperPage,
type ScraperPageConfig,
} from "@/app/scrapers/_components/ScraperPage";
// ── Avito-specific trigger types + hooks ───────────────────────────────────
interface ScrapeAroundResp {
total_fetched: number;
total_inserted: number;
total_updated: number;
by_source: Array<{
source: string;
fetched: number;
inserted: number;
updated: number;
}>;
}
interface HouseTriggerResp {
ok: boolean;
house_url: string;
counters: Record<string, number | string>;
}
interface DetailTriggerResp {
ok: boolean;
item_id: string;
updated: boolean;
}
interface ImvResp {
ok: boolean;
cache_key: string;
recommended_price: number;
range: [number, number];
market_count: number | null;
evaluation_id: number;
history_saved: number;
}
function useScrapeAround() {
return useMutation<
ScrapeAroundResp,
Error,
{ lat: number; lon: number; radius_m: number }
>({
mutationFn: (input) =>
apiFetch<ScrapeAroundResp>("/api/v1/admin/scrape", {
method: "POST",
body: JSON.stringify({ ...input, sources: ["avito"] }),
}),
});
}
function useScrapeHouse() {
return useMutation<HouseTriggerResp, Error, { house_url: string }>({
mutationFn: ({ house_url }) =>
apiFetch<HouseTriggerResp>(
`/api/v1/admin/scrape/avito-house?house_url=${encodeURIComponent(house_url)}`,
{ method: "POST" },
),
});
}
function useScrapeDetail() {
return useMutation<DetailTriggerResp, Error, { item_url: string }>({
mutationFn: ({ item_url }) =>
apiFetch<DetailTriggerResp>(
`/api/v1/admin/scrape/avito-detail?item_url=${encodeURIComponent(item_url)}`,
{ method: "POST" },
),
});
}
function useScrapeImv() {
return useMutation<
ImvResp,
Error,
{
address: string;
rooms: number;
area_m2: number;
floor: number;
floor_at_home: number;
house_type: string;
renovation_type: string;
has_balcony: boolean;
has_loggia: boolean;
}
>({
mutationFn: (input) => {
const qs = new URLSearchParams({
address: input.address,
rooms: String(input.rooms),
area_m2: String(input.area_m2),
floor: String(input.floor),
floor_at_home: String(input.floor_at_home),
house_type: input.house_type,
renovation_type: input.renovation_type,
has_balcony: String(input.has_balcony),
has_loggia: String(input.has_loggia),
});
return apiFetch<ImvResp>(
`/api/v1/admin/scrape/avito-imv?${qs.toString()}`,
{ method: "POST" },
);
},
});
}
// ── Extra triggers ─────────────────────────────────────────────────────────
function AvitoExtraTriggers() {
// 1. Around-point
const [lat, setLat] = useState("56.8400");
const [lon, setLon] = useState("60.6050");
const [radius, setRadius] = useState("1500");
const aroundMut = useScrapeAround();
// 2. Houses Catalog
const [houseUrl, setHouseUrl] = useState(
"/catalog/houses/ekaterinburg/ul_akademika_postovskogo_17a/3171365",
);
const houseMut = useScrapeHouse();
// 3. Detail
const [itemUrl, setItemUrl] = useState(
"/ekaterinburg/kvartiry/3-k._kvartira_75_m_2026_et._7986882804",
);
const detailMut = useScrapeDetail();
// 4. IMV
const [imvAddress, setImvAddress] = useState(
"Свердловская обл., Екатеринбург, ул. Чайковского, 66А",
);
const [imvRooms, setImvRooms] = useState("2");
const [imvArea, setImvArea] = useState("42");
const [imvFloor, setImvFloor] = useState("4");
const [imvTotalFloors, setImvTotalFloors] = useState("5");
const [imvHouseType, setImvHouseType] = useState("panel");
const [imvRenovation, setImvRenovation] = useState("cosmetic");
const [imvBalcony, setImvBalcony] = useState(true);
const [imvLoggia, setImvLoggia] = useState(false);
const imvMut = useScrapeImv();
return (
<>
{/* 1. Search around point */}
<div className="scraper-section">
<h3>1. Search around (общий /admin/scrape)</h3>
<p className="scraper-hint">
Запускает классический cron-pipeline для одного якоря: search
save_listings. Использовать как baseline.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
aroundMut.mutate({
lat: parseFloat(lat),
lon: parseFloat(lon),
radius_m: parseInt(radius, 10),
});
}}
>
<label>
Lat
<input
value={lat}
onChange={(e) => setLat(e.target.value)}
required
/>
</label>
<label>
Lon
<input
value={lon}
onChange={(e) => setLon(e.target.value)}
required
/>
</label>
<label>
Radius (м)
<input
value={radius}
onChange={(e) => setRadius(e.target.value)}
required
/>
</label>
<button type="submit" disabled={aroundMut.isPending}>
{aroundMut.isPending ? "Скрейпим…" : "Запустить"}
</button>
</form>
<ResultPanel mut={aroundMut} />
</div>
{/* 2. Houses Catalog */}
<div className="scraper-section">
<h3>2. Houses Catalog enrichment</h3>
<p className="scraper-hint">
Парсит /catalog/houses/&lt;slug&gt;/&lt;id&gt; характеристики
дома, отзывы, history.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
houseMut.mutate({ house_url: houseUrl });
}}
>
<label className="full">
house_url
<input
value={houseUrl}
onChange={(e) => setHouseUrl(e.target.value)}
required
placeholder="/catalog/houses/<city>/<slug>/<id>"
/>
</label>
<button type="submit" disabled={houseMut.isPending}>
{houseMut.isPending ? "Парсим…" : "Запустить"}
</button>
</form>
<ResultPanel mut={houseMut} />
</div>
{/* 3. Detail */}
<div className="scraper-section">
<h3>3. Detail page enrichment</h3>
<p className="scraper-hint">
Парсит /ekaterinburg/kvartiry/&lt;...&gt; 30+ полей (kitchen_area,
owners_count, etc). UPDATE listings WHERE source_id (listing должен
уже существовать).
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
detailMut.mutate({ item_url: itemUrl });
}}
>
<label className="full">
item_url
<input
value={itemUrl}
onChange={(e) => setItemUrl(e.target.value)}
required
placeholder="/ekaterinburg/kvartiry/.../...."
/>
</label>
<button type="submit" disabled={detailMut.isPending}>
{detailMut.isPending ? "Парсим…" : "Запустить"}
</button>
</form>
<ResultPanel mut={detailMut} />
</div>
{/* 4. IMV evaluation */}
<div className="scraper-section">
<h3>4. IMV evaluation (debug, без cache)</h3>
<p className="scraper-hint">
Прямой вызов Avito IMV API (2 HTTP requests). Production вызов из
estimator on-demand с 24h cache; здесь для debug.
</p>
<form
className="scraper-form"
onSubmit={(e) => {
e.preventDefault();
imvMut.mutate({
address: imvAddress,
rooms: parseInt(imvRooms, 10),
area_m2: parseFloat(imvArea),
floor: parseInt(imvFloor, 10),
floor_at_home: parseInt(imvTotalFloors, 10),
house_type: imvHouseType,
renovation_type: imvRenovation,
has_balcony: imvBalcony,
has_loggia: imvLoggia,
});
}}
>
<label className="full">
Адрес
<input
value={imvAddress}
onChange={(e) => setImvAddress(e.target.value)}
required
/>
</label>
<label>
Комн.
<input
value={imvRooms}
onChange={(e) => setImvRooms(e.target.value)}
required
/>
</label>
<label>
Площадь (м²)
<input
value={imvArea}
onChange={(e) => setImvArea(e.target.value)}
required
/>
</label>
<label>
Этаж
<input
value={imvFloor}
onChange={(e) => setImvFloor(e.target.value)}
required
/>
</label>
<label>
Этажей в доме
<input
value={imvTotalFloors}
onChange={(e) => setImvTotalFloors(e.target.value)}
required
/>
</label>
<label>
Тип дома
<select
value={imvHouseType}
onChange={(e) => setImvHouseType(e.target.value)}
>
<option value="panel">панель</option>
<option value="brick">кирпич</option>
<option value="monolith">монолит</option>
<option value="monolith_brick">монолит-кирпич</option>
<option value="block">блочный</option>
<option value="wood">дерево</option>
</select>
</label>
<label>
Ремонт
<select
value={imvRenovation}
onChange={(e) => setImvRenovation(e.target.value)}
>
<option value="required">требуется</option>
<option value="cosmetic">косметический</option>
<option value="euro">евро</option>
<option value="designer">дизайнерский</option>
</select>
</label>
<label className="checkbox">
<input
type="checkbox"
checked={imvBalcony}
onChange={(e) => setImvBalcony(e.target.checked)}
/>
Балкон
</label>
<label className="checkbox">
<input
type="checkbox"
checked={imvLoggia}
onChange={(e) => setImvLoggia(e.target.checked)}
/>
Лоджия
</label>
<button type="submit" disabled={imvMut.isPending}>
{imvMut.isPending ? "Запрашиваем Avito…" : "Запустить IMV"}
</button>
</form>
<ResultPanel mut={imvMut} />
</div>
</>
);
}
// ── Page config ────────────────────────────────────────────────────────────
const AVITO_CONFIG: ScraperPageConfig = {
source: "avito",
activeTab: "avito",
displayName: "Avito скрапер",
subtitle:
"Ручной запуск sub-pipeline'ов AvitoScraper v2. Для бесплатной разработки и теста после изменений.",
scheduleSource: "avito_city_sweep",
paramConfig: {
hasDetailParams: true,
hasEnrichAddress: false,
defaults: {
pages_per_anchor: 3,
detail_top_n: 20,
request_delay_sec: 7,
radius_m: 1500,
enrich_houses: true,
},
},
delayMin: 3,
delayMax: 30,
};
export default function AvitoScraperPage() {
return (
<ScraperPage
config={{
...AVITO_CONFIG,
extraTriggers: <AvitoExtraTriggers />,
}}
/>
);
}