All checks were successful
Deploy Trade-In / changes (push) Successful in 5s
Deploy Trade-In / test (push) Has been skipped
Deploy Trade-In / build-backend (push) Has been skipped
Deploy Trade-In / build-frontend (push) Successful in 1m37s
Deploy Trade-In / deploy (push) Successful in 36s
Co-authored-by: bot-frontend <bot-frontend@gendsgn.local> Co-committed-by: bot-frontend <bot-frontend@gendsgn.local>
415 lines
12 KiB
TypeScript
415 lines
12 KiB
TypeScript
"use client";
|
||
|
||
import { useState } from "react";
|
||
import { useMutation } from "@tanstack/react-query";
|
||
|
||
import { apiFetch } from "@/lib/api";
|
||
import {
|
||
ResultPanel,
|
||
ScraperPage,
|
||
type ScraperPageConfig,
|
||
} from "@/app/scrapers/_components/ScraperPage";
|
||
|
||
// ── Avito-specific trigger types + hooks ───────────────────────────────────
|
||
|
||
interface ScrapeAroundResp {
|
||
total_fetched: number;
|
||
total_inserted: number;
|
||
total_updated: number;
|
||
by_source: Array<{
|
||
source: string;
|
||
fetched: number;
|
||
inserted: number;
|
||
updated: number;
|
||
}>;
|
||
}
|
||
|
||
interface HouseTriggerResp {
|
||
ok: boolean;
|
||
house_url: string;
|
||
counters: Record<string, number | string>;
|
||
}
|
||
|
||
interface DetailTriggerResp {
|
||
ok: boolean;
|
||
item_id: string;
|
||
updated: boolean;
|
||
}
|
||
|
||
interface ImvResp {
|
||
ok: boolean;
|
||
cache_key: string;
|
||
recommended_price: number;
|
||
range: [number, number];
|
||
market_count: number | null;
|
||
evaluation_id: number;
|
||
history_saved: number;
|
||
}
|
||
|
||
function useScrapeAround() {
|
||
return useMutation<
|
||
ScrapeAroundResp,
|
||
Error,
|
||
{ lat: number; lon: number; radius_m: number }
|
||
>({
|
||
mutationFn: (input) =>
|
||
apiFetch<ScrapeAroundResp>("/api/v1/admin/scrape", {
|
||
method: "POST",
|
||
body: JSON.stringify({ ...input, sources: ["avito"] }),
|
||
}),
|
||
});
|
||
}
|
||
|
||
function useScrapeHouse() {
|
||
return useMutation<HouseTriggerResp, Error, { house_url: string }>({
|
||
mutationFn: ({ house_url }) =>
|
||
apiFetch<HouseTriggerResp>(
|
||
`/api/v1/admin/scrape/avito-house?house_url=${encodeURIComponent(house_url)}`,
|
||
{ method: "POST" },
|
||
),
|
||
});
|
||
}
|
||
|
||
function useScrapeDetail() {
|
||
return useMutation<DetailTriggerResp, Error, { item_url: string }>({
|
||
mutationFn: ({ item_url }) =>
|
||
apiFetch<DetailTriggerResp>(
|
||
`/api/v1/admin/scrape/avito-detail?item_url=${encodeURIComponent(item_url)}`,
|
||
{ method: "POST" },
|
||
),
|
||
});
|
||
}
|
||
|
||
function useScrapeImv() {
|
||
return useMutation<
|
||
ImvResp,
|
||
Error,
|
||
{
|
||
address: string;
|
||
rooms: number;
|
||
area_m2: number;
|
||
floor: number;
|
||
floor_at_home: number;
|
||
house_type: string;
|
||
renovation_type: string;
|
||
has_balcony: boolean;
|
||
has_loggia: boolean;
|
||
}
|
||
>({
|
||
mutationFn: (input) => {
|
||
const qs = new URLSearchParams({
|
||
address: input.address,
|
||
rooms: String(input.rooms),
|
||
area_m2: String(input.area_m2),
|
||
floor: String(input.floor),
|
||
floor_at_home: String(input.floor_at_home),
|
||
house_type: input.house_type,
|
||
renovation_type: input.renovation_type,
|
||
has_balcony: String(input.has_balcony),
|
||
has_loggia: String(input.has_loggia),
|
||
});
|
||
return apiFetch<ImvResp>(
|
||
`/api/v1/admin/scrape/avito-imv?${qs.toString()}`,
|
||
{ method: "POST" },
|
||
);
|
||
},
|
||
});
|
||
}
|
||
|
||
// ── Extra triggers ─────────────────────────────────────────────────────────
|
||
|
||
function AvitoExtraTriggers() {
|
||
// 1. Around-point
|
||
const [lat, setLat] = useState("56.8400");
|
||
const [lon, setLon] = useState("60.6050");
|
||
const [radius, setRadius] = useState("1500");
|
||
const aroundMut = useScrapeAround();
|
||
|
||
// 2. Houses Catalog
|
||
const [houseUrl, setHouseUrl] = useState(
|
||
"/catalog/houses/ekaterinburg/ul_akademika_postovskogo_17a/3171365",
|
||
);
|
||
const houseMut = useScrapeHouse();
|
||
|
||
// 3. Detail
|
||
const [itemUrl, setItemUrl] = useState(
|
||
"/ekaterinburg/kvartiry/3-k._kvartira_75_m_2026_et._7986882804",
|
||
);
|
||
const detailMut = useScrapeDetail();
|
||
|
||
// 4. IMV
|
||
const [imvAddress, setImvAddress] = useState(
|
||
"Свердловская обл., Екатеринбург, ул. Чайковского, 66А",
|
||
);
|
||
const [imvRooms, setImvRooms] = useState("2");
|
||
const [imvArea, setImvArea] = useState("42");
|
||
const [imvFloor, setImvFloor] = useState("4");
|
||
const [imvTotalFloors, setImvTotalFloors] = useState("5");
|
||
const [imvHouseType, setImvHouseType] = useState("panel");
|
||
const [imvRenovation, setImvRenovation] = useState("cosmetic");
|
||
const [imvBalcony, setImvBalcony] = useState(true);
|
||
const [imvLoggia, setImvLoggia] = useState(false);
|
||
const imvMut = useScrapeImv();
|
||
|
||
return (
|
||
<>
|
||
{/* 1. Search around point */}
|
||
<div className="scraper-section">
|
||
<h3>1. Search around (общий /admin/scrape)</h3>
|
||
<p className="scraper-hint">
|
||
Запускает классический cron-pipeline для одного якоря: search →
|
||
save_listings. Использовать как baseline.
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
aroundMut.mutate({
|
||
lat: parseFloat(lat),
|
||
lon: parseFloat(lon),
|
||
radius_m: parseInt(radius, 10),
|
||
});
|
||
}}
|
||
>
|
||
<label>
|
||
Lat
|
||
<input
|
||
value={lat}
|
||
onChange={(e) => setLat(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Lon
|
||
<input
|
||
value={lon}
|
||
onChange={(e) => setLon(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Radius (м)
|
||
<input
|
||
value={radius}
|
||
onChange={(e) => setRadius(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<button type="submit" disabled={aroundMut.isPending}>
|
||
{aroundMut.isPending ? "Скрейпим…" : "Запустить"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={aroundMut} />
|
||
</div>
|
||
|
||
{/* 2. Houses Catalog */}
|
||
<div className="scraper-section">
|
||
<h3>2. Houses Catalog enrichment</h3>
|
||
<p className="scraper-hint">
|
||
Парсит /catalog/houses/<slug>/<id> — характеристики
|
||
дома, отзывы, history.
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
houseMut.mutate({ house_url: houseUrl });
|
||
}}
|
||
>
|
||
<label className="full">
|
||
house_url
|
||
<input
|
||
value={houseUrl}
|
||
onChange={(e) => setHouseUrl(e.target.value)}
|
||
required
|
||
placeholder="/catalog/houses/<city>/<slug>/<id>"
|
||
/>
|
||
</label>
|
||
<button type="submit" disabled={houseMut.isPending}>
|
||
{houseMut.isPending ? "Парсим…" : "Запустить"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={houseMut} />
|
||
</div>
|
||
|
||
{/* 3. Detail */}
|
||
<div className="scraper-section">
|
||
<h3>3. Detail page enrichment</h3>
|
||
<p className="scraper-hint">
|
||
Парсит /ekaterinburg/kvartiry/<...> — 30+ полей (kitchen_area,
|
||
owners_count, etc). UPDATE listings WHERE source_id (listing должен
|
||
уже существовать).
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
detailMut.mutate({ item_url: itemUrl });
|
||
}}
|
||
>
|
||
<label className="full">
|
||
item_url
|
||
<input
|
||
value={itemUrl}
|
||
onChange={(e) => setItemUrl(e.target.value)}
|
||
required
|
||
placeholder="/ekaterinburg/kvartiry/.../...."
|
||
/>
|
||
</label>
|
||
<button type="submit" disabled={detailMut.isPending}>
|
||
{detailMut.isPending ? "Парсим…" : "Запустить"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={detailMut} />
|
||
</div>
|
||
|
||
{/* 4. IMV evaluation */}
|
||
<div className="scraper-section">
|
||
<h3>4. IMV evaluation (debug, без cache)</h3>
|
||
<p className="scraper-hint">
|
||
Прямой вызов Avito IMV API (2 HTTP requests). Production вызов из
|
||
estimator on-demand с 24h cache; здесь — для debug.
|
||
</p>
|
||
<form
|
||
className="scraper-form"
|
||
onSubmit={(e) => {
|
||
e.preventDefault();
|
||
imvMut.mutate({
|
||
address: imvAddress,
|
||
rooms: parseInt(imvRooms, 10),
|
||
area_m2: parseFloat(imvArea),
|
||
floor: parseInt(imvFloor, 10),
|
||
floor_at_home: parseInt(imvTotalFloors, 10),
|
||
house_type: imvHouseType,
|
||
renovation_type: imvRenovation,
|
||
has_balcony: imvBalcony,
|
||
has_loggia: imvLoggia,
|
||
});
|
||
}}
|
||
>
|
||
<label className="full">
|
||
Адрес
|
||
<input
|
||
value={imvAddress}
|
||
onChange={(e) => setImvAddress(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Комн.
|
||
<input
|
||
value={imvRooms}
|
||
onChange={(e) => setImvRooms(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Площадь (м²)
|
||
<input
|
||
value={imvArea}
|
||
onChange={(e) => setImvArea(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Этаж
|
||
<input
|
||
value={imvFloor}
|
||
onChange={(e) => setImvFloor(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Этажей в доме
|
||
<input
|
||
value={imvTotalFloors}
|
||
onChange={(e) => setImvTotalFloors(e.target.value)}
|
||
required
|
||
/>
|
||
</label>
|
||
<label>
|
||
Тип дома
|
||
<select
|
||
value={imvHouseType}
|
||
onChange={(e) => setImvHouseType(e.target.value)}
|
||
>
|
||
<option value="panel">панель</option>
|
||
<option value="brick">кирпич</option>
|
||
<option value="monolith">монолит</option>
|
||
<option value="monolith_brick">монолит-кирпич</option>
|
||
<option value="block">блочный</option>
|
||
<option value="wood">дерево</option>
|
||
</select>
|
||
</label>
|
||
<label>
|
||
Ремонт
|
||
<select
|
||
value={imvRenovation}
|
||
onChange={(e) => setImvRenovation(e.target.value)}
|
||
>
|
||
<option value="required">требуется</option>
|
||
<option value="cosmetic">косметический</option>
|
||
<option value="euro">евро</option>
|
||
<option value="designer">дизайнерский</option>
|
||
</select>
|
||
</label>
|
||
<label className="checkbox">
|
||
<input
|
||
type="checkbox"
|
||
checked={imvBalcony}
|
||
onChange={(e) => setImvBalcony(e.target.checked)}
|
||
/>
|
||
Балкон
|
||
</label>
|
||
<label className="checkbox">
|
||
<input
|
||
type="checkbox"
|
||
checked={imvLoggia}
|
||
onChange={(e) => setImvLoggia(e.target.checked)}
|
||
/>
|
||
Лоджия
|
||
</label>
|
||
<button type="submit" disabled={imvMut.isPending}>
|
||
{imvMut.isPending ? "Запрашиваем Avito…" : "Запустить IMV"}
|
||
</button>
|
||
</form>
|
||
<ResultPanel mut={imvMut} />
|
||
</div>
|
||
</>
|
||
);
|
||
}
|
||
|
||
// ── Page config ────────────────────────────────────────────────────────────
|
||
|
||
const AVITO_CONFIG: ScraperPageConfig = {
|
||
source: "avito",
|
||
activeTab: "avito",
|
||
displayName: "Avito скрапер",
|
||
subtitle:
|
||
"Ручной запуск sub-pipeline'ов AvitoScraper v2. Для бесплатной разработки и теста после изменений.",
|
||
scheduleSource: "avito_city_sweep",
|
||
paramConfig: {
|
||
hasDetailParams: true,
|
||
hasEnrichAddress: false,
|
||
defaults: {
|
||
pages_per_anchor: 3,
|
||
detail_top_n: 20,
|
||
request_delay_sec: 7,
|
||
radius_m: 1500,
|
||
enrich_houses: true,
|
||
},
|
||
},
|
||
delayMin: 3,
|
||
delayMax: 30,
|
||
};
|
||
|
||
export default function AvitoScraperPage() {
|
||
return (
|
||
<ScraperPage
|
||
config={{
|
||
...AVITO_CONFIG,
|
||
extraTriggers: <AvitoExtraTriggers />,
|
||
}}
|
||
/>
|
||
);
|
||
}
|