feat(tradein/cian): admin endpoints + UI for detail+newbuilding history scrape #525
2 changed files with 229 additions and 31 deletions
|
|
@ -931,3 +931,92 @@ async def scrape_yandex_valuation(
|
|||
total_objects=result.house.total_objects,
|
||||
history_items_count=len(result.history_items),
|
||||
)
|
||||
|
||||
|
||||
class CianDetailTriggerResp(BaseModel):
|
||||
ok: bool
|
||||
offer_url: str
|
||||
cian_id: int | None
|
||||
price_changes_count: int
|
||||
saved: bool
|
||||
listing_id: int | None
|
||||
|
||||
|
||||
@router.post("/scrape/cian-detail", response_model=CianDetailTriggerResp)
|
||||
async def scrape_cian_detail(
|
||||
offer_url: str,
|
||||
listing_id: int | None = None,
|
||||
db: Session = Depends(get_db), # noqa: B008
|
||||
) -> CianDetailTriggerResp:
|
||||
"""Ad-hoc parse one Cian offer detail page.
|
||||
|
||||
If `listing_id` provided → save enrichment (offer_price_history + listings updates).
|
||||
Without it → debug-only (no DB write).
|
||||
"""
|
||||
from app.services.scrapers.cian_detail import fetch_detail, save_detail_enrichment
|
||||
|
||||
enrichment = await fetch_detail(offer_url)
|
||||
if enrichment is None:
|
||||
raise HTTPException(404, f"Could not parse Cian detail page: {offer_url}")
|
||||
|
||||
saved = False
|
||||
if listing_id is not None:
|
||||
save_detail_enrichment(db, listing_id, enrichment)
|
||||
saved = True
|
||||
|
||||
return CianDetailTriggerResp(
|
||||
ok=True,
|
||||
offer_url=offer_url,
|
||||
cian_id=enrichment.cian_id,
|
||||
price_changes_count=len(enrichment.price_changes),
|
||||
saved=saved,
|
||||
listing_id=listing_id,
|
||||
)
|
||||
|
||||
|
||||
class CianNewbuildingTriggerResp(BaseModel):
|
||||
ok: bool
|
||||
zhk_url: str
|
||||
cian_internal_house_id: int | None
|
||||
name: str | None
|
||||
price_dynamics_count: int
|
||||
has_management_company: bool
|
||||
saved: bool
|
||||
house_id: int | None
|
||||
|
||||
|
||||
@router.post("/scrape/cian-newbuilding", response_model=CianNewbuildingTriggerResp)
|
||||
async def scrape_cian_newbuilding(
|
||||
zhk_url: str,
|
||||
house_id: int | None = None,
|
||||
db: Session = Depends(get_db), # noqa: B008
|
||||
) -> CianNewbuildingTriggerResp:
|
||||
"""Ad-hoc parse a Cian ЖК (newbuilding) catalog page.
|
||||
|
||||
If `house_id` provided → persist (houses + houses_price_dynamics + management_companies).
|
||||
Without it → debug-only (no DB write).
|
||||
"""
|
||||
from app.services.scrapers.cian_newbuilding import (
|
||||
fetch_newbuilding,
|
||||
save_newbuilding_enrichment,
|
||||
)
|
||||
|
||||
enrichment = await fetch_newbuilding(zhk_url)
|
||||
if enrichment is None:
|
||||
raise HTTPException(404, f"Could not parse Cian newbuilding page: {zhk_url}")
|
||||
|
||||
saved = False
|
||||
if house_id is not None:
|
||||
await save_newbuilding_enrichment(db, house_id, enrichment)
|
||||
saved = True
|
||||
|
||||
return CianNewbuildingTriggerResp(
|
||||
ok=True,
|
||||
zhk_url=zhk_url,
|
||||
cian_internal_house_id=enrichment.cian_internal_house_id,
|
||||
name=enrichment.name,
|
||||
price_dynamics_count=len(enrichment.realty_valuation_chart),
|
||||
has_management_company=bool(enrichment.management_company),
|
||||
saved=saved,
|
||||
house_id=house_id,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -28,6 +28,26 @@ interface TestAuthResp {
|
|||
reason: string | null;
|
||||
}
|
||||
|
||||
interface CianDetailTriggerResp {
|
||||
ok: boolean;
|
||||
offer_url: string;
|
||||
cian_id: number | null;
|
||||
price_changes_count: number;
|
||||
saved: boolean;
|
||||
listing_id: number | null;
|
||||
}
|
||||
|
||||
interface CianNewbuildingTriggerResp {
|
||||
ok: boolean;
|
||||
zhk_url: string;
|
||||
cian_internal_house_id: number | null;
|
||||
name: string | null;
|
||||
price_dynamics_count: number;
|
||||
has_management_company: boolean;
|
||||
saved: boolean;
|
||||
house_id: number | null;
|
||||
}
|
||||
|
||||
interface ScraperSetting {
|
||||
source: string;
|
||||
request_delay_sec: number;
|
||||
|
|
@ -91,6 +111,40 @@ function useUpdateScraperSetting() {
|
|||
});
|
||||
}
|
||||
|
||||
function useScrapeCianDetail() {
|
||||
return useMutation<
|
||||
CianDetailTriggerResp,
|
||||
Error,
|
||||
{ offer_url: string; listing_id: number | undefined }
|
||||
>({
|
||||
mutationFn: ({ offer_url, listing_id }) => {
|
||||
const qs = new URLSearchParams({ offer_url });
|
||||
if (listing_id !== undefined) qs.set("listing_id", String(listing_id));
|
||||
return apiFetch<CianDetailTriggerResp>(
|
||||
`/api/v1/admin/scrape/cian-detail?${qs.toString()}`,
|
||||
{ method: "POST" },
|
||||
);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function useScrapeCianNewbuilding() {
|
||||
return useMutation<
|
||||
CianNewbuildingTriggerResp,
|
||||
Error,
|
||||
{ zhk_url: string; house_id: number | undefined }
|
||||
>({
|
||||
mutationFn: ({ zhk_url, house_id }) => {
|
||||
const qs = new URLSearchParams({ zhk_url });
|
||||
if (house_id !== undefined) qs.set("house_id", String(house_id));
|
||||
return apiFetch<CianNewbuildingTriggerResp>(
|
||||
`/api/v1/admin/scrape/cian-newbuilding?${qs.toString()}`,
|
||||
{ method: "POST" },
|
||||
);
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// ── Result panel ───────────────────────────────────────────────────────────
|
||||
|
||||
interface ResultPanelProps<TData> {
|
||||
|
|
@ -133,6 +187,16 @@ export default function CianScraperPage() {
|
|||
const [multiRoom, setMultiRoom] = useState(false);
|
||||
const aroundMut = useScrapeAround();
|
||||
|
||||
// 2. Detail by URL
|
||||
const [detailUrl, setDetailUrl] = useState("");
|
||||
const [detailListingId, setDetailListingId] = useState("");
|
||||
const detailMut = useScrapeCianDetail();
|
||||
|
||||
// 3. Newbuilding
|
||||
const [nbZhkUrl, setNbZhkUrl] = useState("");
|
||||
const [nbHouseId, setNbHouseId] = useState("");
|
||||
const nbMut = useScrapeCianNewbuilding();
|
||||
|
||||
// Global delay
|
||||
const settingsQ = useScraperSettings();
|
||||
const updateSettingMut = useUpdateScraperSetting();
|
||||
|
|
@ -255,55 +319,100 @@ export default function CianScraperPage() {
|
|||
<ResultPanel mut={aroundMut} />
|
||||
</section>
|
||||
|
||||
{/* 2. Scrape detail by URL — NOT YET IMPLEMENTED BACKEND */}
|
||||
{/* 2. Detail by URL */}
|
||||
<section className="scraper-section">
|
||||
<h2>2. Detail page enrichment</h2>
|
||||
<p className="scraper-hint">
|
||||
Парсинг конкретного объявления Cian по URL. Endpoint{" "}
|
||||
<code>POST /api/v1/admin/scrape/cian-detail</code> планируется в
|
||||
следующем PR. Пока используйте поиск по anchor (#1) для массового
|
||||
скрейпа.
|
||||
Парсинг конкретного объявления Cian по URL: цена, история цен, сохранение
|
||||
в cian_price_dynamics. Если <code>listing_id</code> не указан — только
|
||||
debug-режим без записи в БД.
|
||||
</p>
|
||||
<div
|
||||
className="scraper-result scraper-result--error"
|
||||
style={{ marginTop: 0 }}
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
detailMut.mutate({
|
||||
offer_url: detailUrl,
|
||||
listing_id: detailListingId !== "" ? parseInt(detailListingId, 10) : undefined,
|
||||
});
|
||||
}}
|
||||
>
|
||||
Endpoint не реализован на backend. Запрос вернёт 404.
|
||||
</div>
|
||||
<label className="full">
|
||||
offer_url
|
||||
<input
|
||||
type="url"
|
||||
value={detailUrl}
|
||||
onChange={(e) => setDetailUrl(e.target.value)}
|
||||
required
|
||||
placeholder="https://ekb.cian.ru/sale/flat/XXXXX/"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
listing_id (опционально)
|
||||
<input
|
||||
type="number"
|
||||
value={detailListingId}
|
||||
onChange={(e) => setDetailListingId(e.target.value)}
|
||||
placeholder="оставьте пустым для debug-режима"
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={detailMut.isPending}>
|
||||
{detailMut.isPending ? "Парсим…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={detailMut} />
|
||||
</section>
|
||||
|
||||
{/* 3. Newbuilding — NOT YET IMPLEMENTED BACKEND */}
|
||||
{/* 3. Newbuilding */}
|
||||
<section className="scraper-section">
|
||||
<h2>3. Новостройки (ЖК scrape)</h2>
|
||||
<p className="scraper-hint">
|
||||
Endpoint <code>POST /api/v1/admin/scrape/cian-newbuilding</code>{" "}
|
||||
планируется. Для Yandex Недвижимость аналог уже доступен на вкладке
|
||||
Yandex.
|
||||
Скрейп landing-страницы ЖК Cian: динамика цен, управляющая компания.
|
||||
Если <code>house_id</code> не указан — только debug-режим без записи в БД.
|
||||
</p>
|
||||
<div
|
||||
className="scraper-result scraper-result--error"
|
||||
style={{ marginTop: 0 }}
|
||||
<form
|
||||
className="scraper-form"
|
||||
onSubmit={(e) => {
|
||||
e.preventDefault();
|
||||
nbMut.mutate({
|
||||
zhk_url: nbZhkUrl,
|
||||
house_id: nbHouseId !== "" ? parseInt(nbHouseId, 10) : undefined,
|
||||
});
|
||||
}}
|
||||
>
|
||||
Endpoint не реализован на backend. Запрос вернёт 404.
|
||||
</div>
|
||||
<label className="full">
|
||||
zhk_url
|
||||
<input
|
||||
type="url"
|
||||
value={nbZhkUrl}
|
||||
onChange={(e) => setNbZhkUrl(e.target.value)}
|
||||
required
|
||||
placeholder="https://zhk-XXX-ekb-i.cian.ru/"
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
house_id (опционально)
|
||||
<input
|
||||
type="number"
|
||||
value={nbHouseId}
|
||||
onChange={(e) => setNbHouseId(e.target.value)}
|
||||
placeholder="оставьте пустым для debug-режима"
|
||||
/>
|
||||
</label>
|
||||
<button type="submit" disabled={nbMut.isPending}>
|
||||
{nbMut.isPending ? "Парсим…" : "Запустить"}
|
||||
</button>
|
||||
</form>
|
||||
<ResultPanel mut={nbMut} />
|
||||
</section>
|
||||
|
||||
{/* 4. Valuation Calculator — NOT YET IMPLEMENTED BACKEND */}
|
||||
{/* 4. Valuation Calculator */}
|
||||
<section className="scraper-section">
|
||||
<h2>4. Valuation Calculator (оценка квартиры)</h2>
|
||||
<p className="scraper-hint">
|
||||
Endpoint <code>POST /api/v1/admin/scrape/cian-valuation</code>{" "}
|
||||
планируется. Требует загруженных cookies (Stage 7).{" "}
|
||||
<Link href="/scrapers/cian-cookies">
|
||||
Настроить cookies →
|
||||
</Link>
|
||||
Calculator вызывается автоматически из /estimate flow (Stage 9).
|
||||
Отдельный admin trigger будет добавлен позже.
|
||||
</p>
|
||||
<div
|
||||
className="scraper-result scraper-result--error"
|
||||
style={{ marginTop: 0 }}
|
||||
>
|
||||
Endpoint не реализован на backend. Запрос вернёт 404.
|
||||
</div>
|
||||
</section>
|
||||
|
||||
{/* 5. Global Request Delay */}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue