79070bfe99
- Discovered real endpoints via chrome-devtools: /сервисы/api/kn/object (offset/limit/place/objStatus) + /portal-kn/api/sales/portal/table. Server-side filter ignored — local filter by developer.companyGroup. - Pure Playwright in-context fetch (httpx blocked by ServicePipe TLS-fp). - Saved fingerprint state in data/playwright_state.json for server reuse. - Celery beat (configurable cron + jitter) + admin trigger endpoint + /admin/scrape UI page. - Schema migration: UNIQUE(id, snapshot_date) for versioned snapshots, kn_scrape_runs journal. - Smoke-tested on PRINZIP: 28 ЖК / 826 квартир as expected.
101 lines
3.3 KiB
Python
101 lines
3.3 KiB
Python
"""Admin trigger for ad-hoc kn-API sweeps.
|
|
|
|
POST /api/v1/admin/scrape/kn
|
|
Body: {"region_code": 66, "developers": ["6208_0"]?, "fetch_flats": true?}
|
|
Header: X-Admin-Token: <SCRAPE_ADMIN_TOKEN env var>
|
|
|
|
Disabled if SCRAPE_ADMIN_TOKEN is empty (default).
|
|
Returns the Celery task id; poll /api/v1/admin/scrape/runs/{run_id} to track DB-side progress.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import Annotated, Any
|
|
|
|
from fastapi import APIRouter, Depends, Header, HTTPException
|
|
from pydantic import BaseModel, Field
|
|
from sqlalchemy import text
|
|
from sqlalchemy.orm import Session
|
|
|
|
from app.core.config import settings
|
|
from app.core.db import get_db
|
|
|
|
router = APIRouter()
|
|
|
|
|
|
class TriggerKnRequest(BaseModel):
|
|
region_code: int = Field(..., ge=1, le=99)
|
|
developers: list[str] | None = Field(default=None, max_length=20)
|
|
fetch_flats: bool = True
|
|
|
|
|
|
def _check_token(x_admin_token: str | None) -> None:
|
|
if not settings.scrape_admin_token:
|
|
raise HTTPException(
|
|
status_code=503,
|
|
detail="admin scrape disabled — set SCRAPE_ADMIN_TOKEN",
|
|
)
|
|
if x_admin_token != settings.scrape_admin_token:
|
|
raise HTTPException(status_code=401, detail="invalid admin token")
|
|
|
|
|
|
@router.post("/kn")
|
|
def trigger_kn_sweep(
|
|
payload: TriggerKnRequest,
|
|
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
|
) -> dict[str, Any]:
|
|
_check_token(x_admin_token)
|
|
# Defer Celery import so the API works without a broker in dev.
|
|
from app.workers.tasks.scrape_kn import scrape_kn_region
|
|
|
|
result = scrape_kn_region.apply_async(
|
|
args=[payload.region_code, payload.developers],
|
|
kwargs={"skip_jitter": True},
|
|
)
|
|
return {
|
|
"task_id": result.id,
|
|
"region_code": payload.region_code,
|
|
"developers": payload.developers,
|
|
"queued_at": result.date_done.isoformat() if result.date_done else None,
|
|
}
|
|
|
|
|
|
@router.get("/runs")
|
|
def list_runs(
|
|
db: Annotated[Session, Depends(get_db)],
|
|
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
|
limit: int = 20,
|
|
) -> list[dict[str, Any]]:
|
|
_check_token(x_admin_token)
|
|
rows = (
|
|
db.execute(
|
|
text(
|
|
"""
|
|
SELECT run_id, started_at, finished_at, region_codes, developer_ids,
|
|
objects_count, flats_count, requests_count, status, error, snapshot_date
|
|
FROM kn_scrape_runs
|
|
ORDER BY started_at DESC
|
|
LIMIT :lim
|
|
"""
|
|
),
|
|
{"lim": min(limit, 100)},
|
|
)
|
|
.mappings()
|
|
.all()
|
|
)
|
|
return [
|
|
{
|
|
"run_id": r["run_id"],
|
|
"started_at": r["started_at"].isoformat() if r["started_at"] else None,
|
|
"finished_at": r["finished_at"].isoformat() if r["finished_at"] else None,
|
|
"region_codes": list(r["region_codes"]) if r["region_codes"] else [],
|
|
"developer_ids": list(r["developer_ids"]) if r["developer_ids"] else [],
|
|
"objects_count": r["objects_count"],
|
|
"flats_count": r["flats_count"],
|
|
"requests_count": r["requests_count"],
|
|
"status": r["status"],
|
|
"error": r["error"],
|
|
"snapshot_date": r["snapshot_date"].isoformat() if r["snapshot_date"] else None,
|
|
}
|
|
for r in rows
|
|
]
|