add domrf extras parser (sale_graph + sales_agg + infra + photos) +
fix Docker build for playwright - Regenerate uv.lock with playwright + tenacity (was missing → uv sync --frozen skipped them, hence playwright not in /app/.venv/bin/). - Dockerfile: explicit Chromium runtime libs in apt (libnss3 + 14 more) + `playwright install chromium` BEFORE apt-lists cleanup. - Schema 51_schema_kn_extras.sql: 5 new tables for time-series sales, current aggregates, POI infrastructure, photo metadata, failure log. - Scraper: 4 new fetch helpers + upserts; photo binary download via Playwright APIRequest; failure-logging with full_url to kn_scrape_failures. - CLI: --no-extras / --download-photos / --no-flats / --probe URL. - Admin UI: extras checkboxes + failures table with copy-URL button. - PRINZIP smoke: 28 objs, 5409 POI, 5150 photos metadata, sales_agg matches probe (Парк Победы 145/291 = 49%).
This commit is contained in:
lekss361
parent
621d56c064
commit
6d3fa8cbd2
8 changed files with 977 additions and 13 deletions
|
|
@ -27,6 +27,8 @@ class TriggerKnRequest(BaseModel):
|
|||
region_code: int = Field(..., ge=1, le=99)
|
||||
developers: list[str] | None = Field(default=None, max_length=20)
|
||||
fetch_flats: bool = True
|
||||
extras: bool = True
|
||||
download_photos: bool = False
|
||||
|
||||
|
||||
def _check_token(x_admin_token: str | None) -> None:
|
||||
|
|
@ -50,7 +52,12 @@ def trigger_kn_sweep(
|
|||
|
||||
result = scrape_kn_region.apply_async(
|
||||
args=[payload.region_code, payload.developers],
|
||||
kwargs={"skip_jitter": True},
|
||||
kwargs={
|
||||
"skip_jitter": True,
|
||||
"extras": payload.extras,
|
||||
"download_photos": payload.download_photos,
|
||||
"fetch_flats": payload.fetch_flats,
|
||||
},
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
|
|
@ -60,6 +67,53 @@ def trigger_kn_sweep(
|
|||
}
|
||||
|
||||
|
||||
@router.get("/failures")
|
||||
def list_failures(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||
run_id: int | None = None,
|
||||
limit: int = 50,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Per-request failure log for manual browser verification."""
|
||||
_check_token(x_admin_token)
|
||||
where = ""
|
||||
params: dict[str, Any] = {"lim": min(limit, 200)}
|
||||
if run_id is not None:
|
||||
where = "WHERE run_id = :rid"
|
||||
params["rid"] = run_id
|
||||
rows = (
|
||||
db.execute(
|
||||
text(
|
||||
f"""
|
||||
SELECT failure_id, run_id, obj_id, endpoint, full_url,
|
||||
http_status, error, body_preview, occurred_at
|
||||
FROM kn_scrape_failures
|
||||
{where}
|
||||
ORDER BY occurred_at DESC
|
||||
LIMIT :lim
|
||||
"""
|
||||
),
|
||||
params,
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"failure_id": r["failure_id"],
|
||||
"run_id": r["run_id"],
|
||||
"obj_id": r["obj_id"],
|
||||
"endpoint": r["endpoint"],
|
||||
"full_url": r["full_url"],
|
||||
"http_status": r["http_status"],
|
||||
"error": r["error"],
|
||||
"body_preview": r["body_preview"],
|
||||
"occurred_at": r["occurred_at"].isoformat() if r["occurred_at"] else None,
|
||||
}
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
@router.get("/runs")
|
||||
def list_runs(
|
||||
db: Annotated[Session, Depends(get_db)],
|
||||
|
|
|
|||
|
|
@ -19,20 +19,28 @@ from __future__ import annotations
|
|||
import json
|
||||
import logging
|
||||
from datetime import date, datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
from app.services.scrapers.stealth import BrowserSession
|
||||
from app.services.scrapers.stealth import BASE_URL, BrowserSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
PATH_OBJECTS = "/сервисы/api/kn/object"
|
||||
PATH_FLATS_TABLE = "/portal-kn/api/sales/portal/table"
|
||||
PATH_SALE_GRAPH = "/сервисы/api/object/{obj_id}/sale_graph" # ?type=apartments|parking
|
||||
PATH_SALES_AGG = "/сервисы/api/object/{obj_id}/sales_agg"
|
||||
PATH_INFRA = "/сервисы/api/object/{obj_id}/infrastructure"
|
||||
PATH_PHOTOS = "/сервисы/api/object/construction/progress/photo/{obj_id}"
|
||||
RAW_SECTION = "kn_api"
|
||||
|
||||
SALE_GRAPH_TYPES = ("apartments", "parking")
|
||||
PHOTOS_DIR_DEFAULT = Path("data/raw/domrf_photos")
|
||||
|
||||
# Three statuses observed in production (Apr 2026):
|
||||
# 0 — строящиеся ЖК
|
||||
# 1 — небольшая выборка ~2 объекта на регион (legacy/archive flag)
|
||||
|
|
@ -367,6 +375,367 @@ def upsert_flats(
|
|||
return inserted
|
||||
|
||||
|
||||
# ── extras (sale_graph, sales_agg, infrastructure, photos) ──────────────────
|
||||
|
||||
UPSERT_SALE_GRAPH_SQL = text(
|
||||
"""
|
||||
INSERT INTO domrf_kn_sale_graph (
|
||||
obj_id, report_month, type, realised, contracted, area_sq, price_avg, snapshot_date
|
||||
) VALUES (
|
||||
:obj_id, :report_month, :type, :realised, :contracted, :area_sq, :price_avg, :snapshot_date
|
||||
)
|
||||
ON CONFLICT (obj_id, report_month, type, snapshot_date) DO UPDATE SET
|
||||
realised = EXCLUDED.realised,
|
||||
contracted = EXCLUDED.contracted,
|
||||
area_sq = EXCLUDED.area_sq,
|
||||
price_avg = EXCLUDED.price_avg
|
||||
"""
|
||||
)
|
||||
|
||||
UPSERT_SALES_AGG_SQL = text(
|
||||
"""
|
||||
INSERT INTO domrf_kn_sales_agg (
|
||||
obj_id, type, name, total, realised, perc, snapshot_date
|
||||
) VALUES (
|
||||
:obj_id, :type, :name, :total, :realised, :perc, :snapshot_date
|
||||
)
|
||||
ON CONFLICT (obj_id, type, snapshot_date) DO UPDATE SET
|
||||
name = EXCLUDED.name,
|
||||
total = EXCLUDED.total,
|
||||
realised = EXCLUDED.realised,
|
||||
perc = EXCLUDED.perc
|
||||
"""
|
||||
)
|
||||
|
||||
UPSERT_INFRA_SQL = text(
|
||||
"""
|
||||
INSERT INTO domrf_kn_infrastructure (
|
||||
obj_id, poi_name, poi_subtitle, poi_category, poi_address,
|
||||
poi_lat, poi_lon, distance_m, snapshot_date
|
||||
) VALUES (
|
||||
:obj_id, :poi_name, :poi_subtitle, :poi_category, :poi_address,
|
||||
:poi_lat, :poi_lon, :distance_m, :snapshot_date
|
||||
)
|
||||
ON CONFLICT (obj_id, poi_name, poi_lat, poi_lon, snapshot_date) DO UPDATE SET
|
||||
poi_subtitle = EXCLUDED.poi_subtitle,
|
||||
poi_category = EXCLUDED.poi_category,
|
||||
poi_address = EXCLUDED.poi_address,
|
||||
distance_m = EXCLUDED.distance_m
|
||||
"""
|
||||
)
|
||||
|
||||
UPSERT_PHOTO_SQL = text(
|
||||
"""
|
||||
INSERT INTO domrf_kn_photos (
|
||||
obj_id, obj_file_id, ord_num, photo_url, photo_dttm, period_dt,
|
||||
size_bytes, photo_name, ready_desc, build_type, hidden,
|
||||
local_path, downloaded_at
|
||||
) VALUES (
|
||||
:obj_id, :obj_file_id, :ord_num, :photo_url, :photo_dttm, :period_dt,
|
||||
:size_bytes, :photo_name, :ready_desc, :build_type, :hidden,
|
||||
:local_path, :downloaded_at
|
||||
)
|
||||
ON CONFLICT (obj_id, obj_file_id) DO UPDATE SET
|
||||
ord_num = EXCLUDED.ord_num,
|
||||
photo_url = EXCLUDED.photo_url,
|
||||
photo_dttm = EXCLUDED.photo_dttm,
|
||||
period_dt = EXCLUDED.period_dt,
|
||||
size_bytes = EXCLUDED.size_bytes,
|
||||
photo_name = EXCLUDED.photo_name,
|
||||
ready_desc = EXCLUDED.ready_desc,
|
||||
hidden = EXCLUDED.hidden,
|
||||
local_path = COALESCE(EXCLUDED.local_path, domrf_kn_photos.local_path),
|
||||
downloaded_at = COALESCE(EXCLUDED.downloaded_at, domrf_kn_photos.downloaded_at)
|
||||
"""
|
||||
)
|
||||
|
||||
INSERT_FAILURE_SQL = text(
|
||||
"""
|
||||
INSERT INTO kn_scrape_failures (
|
||||
run_id, obj_id, endpoint, full_url, http_status, error, body_preview
|
||||
) VALUES (:run_id, :obj_id, :endpoint, :full_url, :http_status, :error, :body_preview)
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _to_dt(s: Any) -> datetime | None:
|
||||
"""Coerce 'DD-MM-YYYY HH:MM:SS' or 'YYYY-MM-DD ...' to datetime."""
|
||||
if not s or not isinstance(s, str):
|
||||
return None
|
||||
for fmt in ("%d-%m-%Y %H:%M:%S", "%Y-%m-%d %H:%M:%S", "%d-%m-%Y", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(s.strip(), fmt)
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _to_month_date(s: Any) -> date | None:
|
||||
"""Coerce '01.04.2025' (sale_graph reportMonthDt) to date."""
|
||||
if not s or not isinstance(s, str):
|
||||
return None
|
||||
for fmt in ("%d.%m.%Y", "%Y-%m-%d"):
|
||||
try:
|
||||
return datetime.strptime(s.strip(), fmt).date()
|
||||
except ValueError:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _http_status_of(e: Exception) -> int | None:
|
||||
"""Try to extract HTTP status from exception message like 'http 404: ...'."""
|
||||
msg = str(e)
|
||||
if msg.startswith("http "):
|
||||
try:
|
||||
return int(msg.split()[1].rstrip(":"))
|
||||
except (IndexError, ValueError):
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _body_of(e: Exception) -> str:
|
||||
"""Extract body preview from exception message."""
|
||||
msg = str(e)
|
||||
if ":" in msg:
|
||||
return msg.split(":", 1)[1].strip()[:500]
|
||||
return msg[:500]
|
||||
|
||||
|
||||
def _log_failure(
|
||||
db: Session,
|
||||
run_id: int | None,
|
||||
obj_id: int | None,
|
||||
endpoint: str,
|
||||
full_url: str,
|
||||
http_status: int | None,
|
||||
error: str,
|
||||
body_preview: str = "",
|
||||
) -> None:
|
||||
"""Record a per-request failure into kn_scrape_failures.
|
||||
|
||||
Caller commits later — this just stages the INSERT in the current transaction.
|
||||
Uses a SAVEPOINT so a serialisation issue here doesn't poison the outer txn.
|
||||
"""
|
||||
if run_id is None:
|
||||
return
|
||||
try:
|
||||
db.execute(
|
||||
INSERT_FAILURE_SQL,
|
||||
{
|
||||
"run_id": run_id,
|
||||
"obj_id": obj_id,
|
||||
"endpoint": endpoint[:255] if endpoint else None,
|
||||
"full_url": full_url[:1000] if full_url else None,
|
||||
"http_status": http_status,
|
||||
"error": error[:1000] if error else None,
|
||||
"body_preview": body_preview[:500] if body_preview else None,
|
||||
},
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("failed to log failure: %s", e)
|
||||
db.rollback()
|
||||
|
||||
|
||||
async def fetch_sale_graph(
|
||||
sess: BrowserSession, obj_id: int, type_: str
|
||||
) -> tuple[list[dict[str, Any]], str]:
|
||||
"""Return (rows, full_url). On HTTP errors raises — caller logs."""
|
||||
path = PATH_SALE_GRAPH.format(obj_id=obj_id)
|
||||
payload = await sess.get_json(path, {"type": type_})
|
||||
full_url = f"{BASE_URL}{path}?type={type_}"
|
||||
rows = (payload.get("data") or {}).get("salesGraphDtos") or []
|
||||
return rows, full_url
|
||||
|
||||
|
||||
async def fetch_sales_agg(sess: BrowserSession, obj_id: int) -> tuple[dict[str, Any], str]:
|
||||
path = PATH_SALES_AGG.format(obj_id=obj_id)
|
||||
payload = await sess.get_json(path, {})
|
||||
return (payload.get("data") or {}), f"{BASE_URL}{path}"
|
||||
|
||||
|
||||
async def fetch_infrastructure(
|
||||
sess: BrowserSession, obj_id: int
|
||||
) -> tuple[list[dict[str, Any]], str]:
|
||||
path = PATH_INFRA.format(obj_id=obj_id)
|
||||
payload = await sess.get_json(path, {})
|
||||
return (payload.get("data") or []), f"{BASE_URL}{path}"
|
||||
|
||||
|
||||
async def fetch_photos(sess: BrowserSession, obj_id: int) -> tuple[list[dict[str, Any]], str]:
|
||||
path = PATH_PHOTOS.format(obj_id=obj_id)
|
||||
# /photo endpoint returns top-level array, not {data: ...}
|
||||
payload = await sess.get_json(path, {})
|
||||
full_url = f"{BASE_URL}{path}"
|
||||
if isinstance(payload, list):
|
||||
return payload, full_url
|
||||
return (payload.get("data") or []), full_url
|
||||
|
||||
|
||||
def upsert_sale_graph(
|
||||
db: Session, obj_id: int, type_: str, rows: list[dict[str, Any]], snapshot_date: date
|
||||
) -> int:
|
||||
n = 0
|
||||
for r in rows:
|
||||
rm = _to_month_date(r.get("reportMonthDt"))
|
||||
if rm is None:
|
||||
continue
|
||||
try:
|
||||
db.execute(
|
||||
UPSERT_SALE_GRAPH_SQL,
|
||||
{
|
||||
"obj_id": obj_id,
|
||||
"report_month": rm,
|
||||
"type": type_,
|
||||
"realised": r.get("realised"),
|
||||
"contracted": r.get("contracted"),
|
||||
"area_sq": r.get("areaSq"),
|
||||
"price_avg": r.get("priceAvg"),
|
||||
"snapshot_date": snapshot_date,
|
||||
},
|
||||
)
|
||||
n += 1
|
||||
except Exception as e:
|
||||
logger.warning("upsert sale_graph obj=%s type=%s month=%s: %s", obj_id, type_, rm, e)
|
||||
db.rollback()
|
||||
return n
|
||||
|
||||
|
||||
def upsert_sales_agg(db: Session, obj_id: int, agg: dict[str, Any], snapshot_date: date) -> int:
|
||||
"""agg is {apartmentsAggData: {...}, nonlivAggData: {...}, parkingAggData: {...}}."""
|
||||
type_map = {
|
||||
"apartmentsAggData": "apartments",
|
||||
"nonlivAggData": "nonliv",
|
||||
"parkingAggData": "parking",
|
||||
}
|
||||
n = 0
|
||||
for key, type_ in type_map.items():
|
||||
d = agg.get(key)
|
||||
if not isinstance(d, dict):
|
||||
continue
|
||||
try:
|
||||
db.execute(
|
||||
UPSERT_SALES_AGG_SQL,
|
||||
{
|
||||
"obj_id": obj_id,
|
||||
"type": type_,
|
||||
"name": d.get("name"),
|
||||
"total": d.get("total"),
|
||||
"realised": d.get("realised"),
|
||||
"perc": d.get("perc"),
|
||||
"snapshot_date": snapshot_date,
|
||||
},
|
||||
)
|
||||
n += 1
|
||||
except Exception as e:
|
||||
logger.warning("upsert sales_agg obj=%s type=%s: %s", obj_id, type_, e)
|
||||
db.rollback()
|
||||
return n
|
||||
|
||||
|
||||
def upsert_infrastructure(
|
||||
db: Session, obj_id: int, pois: list[dict[str, Any]], snapshot_date: date
|
||||
) -> int:
|
||||
n = 0
|
||||
for poi in pois:
|
||||
try:
|
||||
db.execute(
|
||||
UPSERT_INFRA_SQL,
|
||||
{
|
||||
"obj_id": obj_id,
|
||||
"poi_name": (poi.get("name") or "").strip()[:500] or "—",
|
||||
"poi_subtitle": poi.get("subtitle"),
|
||||
"poi_category": poi.get("tabName"),
|
||||
"poi_address": poi.get("address"),
|
||||
"poi_lat": poi.get("latitude"),
|
||||
"poi_lon": poi.get("longitude"),
|
||||
"distance_m": poi.get("dist"),
|
||||
"snapshot_date": snapshot_date,
|
||||
},
|
||||
)
|
||||
n += 1
|
||||
except Exception as e:
|
||||
logger.warning("upsert infra obj=%s poi=%s: %s", obj_id, poi.get("name"), e)
|
||||
db.rollback()
|
||||
return n
|
||||
|
||||
|
||||
def upsert_photos(
|
||||
db: Session,
|
||||
obj_id: int,
|
||||
photos: list[dict[str, Any]],
|
||||
local_paths: dict[str, str] | None = None,
|
||||
) -> int:
|
||||
"""Upsert photo metadata. local_paths maps file_id → relative path on disk."""
|
||||
local_paths = local_paths or {}
|
||||
now_dt = datetime.now()
|
||||
n = 0
|
||||
for p in photos:
|
||||
file_id = p.get("objFileId")
|
||||
if not file_id:
|
||||
continue
|
||||
local = local_paths.get(file_id)
|
||||
try:
|
||||
db.execute(
|
||||
UPSERT_PHOTO_SQL,
|
||||
{
|
||||
"obj_id": obj_id,
|
||||
"obj_file_id": file_id,
|
||||
"ord_num": p.get("ordNum"),
|
||||
"photo_url": p.get("objPhotoUrl"),
|
||||
"photo_dttm": _to_dt(p.get("objPhotoDttm")),
|
||||
"period_dt": _to_month_date(p.get("objPeriodDt")),
|
||||
"size_bytes": p.get("objSize"),
|
||||
"photo_name": p.get("objPhotoNm"),
|
||||
"ready_desc": p.get("objReadyDesc"),
|
||||
"build_type": p.get("objBuildTypeShortDesc"),
|
||||
"hidden": _to_bool(p.get("objPhotoHiddenFlg")),
|
||||
"local_path": local,
|
||||
"downloaded_at": now_dt if local else None,
|
||||
},
|
||||
)
|
||||
n += 1
|
||||
except Exception as e:
|
||||
logger.warning("upsert photo obj=%s file=%s: %s", obj_id, file_id, e)
|
||||
db.rollback()
|
||||
return n
|
||||
|
||||
|
||||
async def download_photos(
|
||||
sess: BrowserSession,
|
||||
obj_id: int,
|
||||
photos: list[dict[str, Any]],
|
||||
base_dir: Path,
|
||||
) -> dict[str, str]:
|
||||
"""Download photo binaries to base_dir/{obj_id}/{file_id}.png (or .jpg).
|
||||
Returns mapping {file_id: relative_path}. Skips if file already exists with
|
||||
matching size_bytes.
|
||||
"""
|
||||
out: dict[str, str] = {}
|
||||
obj_dir = base_dir / str(obj_id)
|
||||
obj_dir.mkdir(parents=True, exist_ok=True)
|
||||
for p in photos:
|
||||
file_id = p.get("objFileId")
|
||||
url = p.get("objPhotoUrl")
|
||||
if not file_id or not url:
|
||||
continue
|
||||
# Extension hint from objPhotoNm or default png
|
||||
name = (p.get("objPhotoNm") or "").lower()
|
||||
ext = ".jpg" if name.endswith((".jpg", ".jpeg")) else ".png"
|
||||
local = obj_dir / f"{file_id}{ext}"
|
||||
# Skip if size matches
|
||||
expected_size = p.get("objSize")
|
||||
if local.exists() and expected_size and local.stat().st_size == expected_size:
|
||||
out[file_id] = str(local)
|
||||
continue
|
||||
try:
|
||||
data = await sess.download_binary(url)
|
||||
local.write_bytes(data)
|
||||
out[file_id] = str(local)
|
||||
except Exception as e:
|
||||
logger.warning("download photo obj=%s file=%s failed: %s", obj_id, file_id, e)
|
||||
return out
|
||||
|
||||
|
||||
# ── orchestrator ─────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
|
|
@ -410,6 +779,55 @@ async def fetch_flats_for_object(sess: BrowserSession, obj_id: int) -> list[dict
|
|||
return _flatten_table(payload)
|
||||
|
||||
|
||||
async def probe_endpoint(
|
||||
url_or_path: str,
|
||||
region_code: int = 66,
|
||||
headed: bool = False,
|
||||
load_state: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Quick one-shot probe through a real Playwright session for debugging.
|
||||
|
||||
Returns {status, content_type, body_preview, full_url}. Use to verify that
|
||||
a URL works in our session before/after a failing scraper run.
|
||||
"""
|
||||
if url_or_path.startswith("http"):
|
||||
# split into path+query for BrowserSession.get_json
|
||||
from urllib.parse import parse_qsl, urlsplit
|
||||
|
||||
s = urlsplit(url_or_path)
|
||||
path = s.path
|
||||
params = dict(parse_qsl(s.query, keep_blank_values=True))
|
||||
else:
|
||||
path = url_or_path.split("?", 1)[0]
|
||||
params = {}
|
||||
if "?" in url_or_path:
|
||||
from urllib.parse import parse_qsl
|
||||
|
||||
params = dict(parse_qsl(url_or_path.split("?", 1)[1], keep_blank_values=True))
|
||||
|
||||
async with BrowserSession(
|
||||
region_code=region_code, headed=headed, load_state=load_state
|
||||
) as sess:
|
||||
try:
|
||||
payload = await sess.get_json(path, params)
|
||||
body = json.dumps(payload, ensure_ascii=False)[:2000]
|
||||
qs = "&".join(f"{k}={v}" for k, v in params.items())
|
||||
full_url = f"{BASE_URL}{path}" + (f"?{qs}" if qs else "")
|
||||
return {
|
||||
"status": 200,
|
||||
"content_type": "application/json",
|
||||
"body_preview": body,
|
||||
"full_url": full_url,
|
||||
}
|
||||
except Exception as e:
|
||||
return {
|
||||
"status": _http_status_of(e),
|
||||
"content_type": "error",
|
||||
"body_preview": str(e)[:2000],
|
||||
"full_url": f"{BASE_URL}{path}",
|
||||
}
|
||||
|
||||
|
||||
async def run_region_sweep(
|
||||
region_code: int,
|
||||
developers: list[str] | None = None,
|
||||
|
|
@ -419,6 +837,9 @@ async def run_region_sweep(
|
|||
save_state: str | None = None,
|
||||
statuses: tuple[int, ...] = DEFAULT_STATUSES,
|
||||
fetch_flats: bool = True,
|
||||
extras: bool = True,
|
||||
download_photos_binary: bool = False,
|
||||
photos_dir: Path | str | None = None,
|
||||
place_override: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Full sweep for one region: bootstrap → fetch objects (each objStatus) → fetch flats.
|
||||
|
|
@ -489,6 +910,127 @@ async def run_region_sweep(
|
|||
)
|
||||
logger.info("total flats: %d (across %d objs)", len(all_flats), len(all_objects))
|
||||
|
||||
# --- extras (sale_graph, sales_agg, infra, photos) per object ----
|
||||
extras_counts = {
|
||||
"sale_graph_rows": 0,
|
||||
"sales_agg_rows": 0,
|
||||
"infra_rows": 0,
|
||||
"photos_rows": 0,
|
||||
"photos_downloaded": 0,
|
||||
}
|
||||
if extras:
|
||||
pdir = Path(photos_dir) if photos_dir else PHOTOS_DIR_DEFAULT
|
||||
for i, obj in enumerate(all_objects):
|
||||
obj_id = obj.get("objId") or obj.get("obj_id")
|
||||
if not obj_id:
|
||||
continue
|
||||
|
||||
# sale_graph (apartments + parking)
|
||||
for type_ in SALE_GRAPH_TYPES:
|
||||
try:
|
||||
rows, full_url = await fetch_sale_graph(sess, obj_id, type_)
|
||||
extras_counts["sale_graph_rows"] += upsert_sale_graph(
|
||||
db, obj_id, type_, rows, snapshot_date
|
||||
)
|
||||
except Exception as e:
|
||||
full_url = (
|
||||
f"{BASE_URL}{PATH_SALE_GRAPH.format(obj_id=obj_id)}?type={type_}"
|
||||
)
|
||||
logger.warning(
|
||||
"sale_graph obj=%s type=%s FAILED: %s\n URL: %s",
|
||||
obj_id,
|
||||
type_,
|
||||
e,
|
||||
full_url,
|
||||
)
|
||||
_log_failure(
|
||||
db,
|
||||
run_id,
|
||||
obj_id,
|
||||
f"sale_graph_{type_}",
|
||||
full_url,
|
||||
_http_status_of(e),
|
||||
str(e),
|
||||
_body_of(e),
|
||||
)
|
||||
|
||||
# sales_agg
|
||||
try:
|
||||
agg, full_url = await fetch_sales_agg(sess, obj_id)
|
||||
extras_counts["sales_agg_rows"] += upsert_sales_agg(
|
||||
db, obj_id, agg, snapshot_date
|
||||
)
|
||||
except Exception as e:
|
||||
full_url = f"{BASE_URL}{PATH_SALES_AGG.format(obj_id=obj_id)}"
|
||||
logger.warning(
|
||||
"sales_agg obj=%s FAILED: %s\n URL: %s", obj_id, e, full_url
|
||||
)
|
||||
_log_failure(
|
||||
db,
|
||||
run_id,
|
||||
obj_id,
|
||||
"sales_agg",
|
||||
full_url,
|
||||
_http_status_of(e),
|
||||
str(e),
|
||||
_body_of(e),
|
||||
)
|
||||
|
||||
# infrastructure
|
||||
try:
|
||||
pois, full_url = await fetch_infrastructure(sess, obj_id)
|
||||
extras_counts["infra_rows"] += upsert_infrastructure(
|
||||
db, obj_id, pois, snapshot_date
|
||||
)
|
||||
except Exception as e:
|
||||
full_url = f"{BASE_URL}{PATH_INFRA.format(obj_id=obj_id)}"
|
||||
logger.warning("infra obj=%s FAILED: %s\n URL: %s", obj_id, e, full_url)
|
||||
_log_failure(
|
||||
db,
|
||||
run_id,
|
||||
obj_id,
|
||||
"infrastructure",
|
||||
full_url,
|
||||
_http_status_of(e),
|
||||
str(e),
|
||||
_body_of(e),
|
||||
)
|
||||
|
||||
# photos (metadata + optional binary)
|
||||
try:
|
||||
photos, full_url = await fetch_photos(sess, obj_id)
|
||||
local_paths: dict[str, str] = {}
|
||||
if download_photos_binary and photos:
|
||||
local_paths = await download_photos(sess, obj_id, photos, pdir)
|
||||
extras_counts["photos_downloaded"] += len(local_paths)
|
||||
extras_counts["photos_rows"] += upsert_photos(
|
||||
db, obj_id, photos, local_paths
|
||||
)
|
||||
except Exception as e:
|
||||
full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}"
|
||||
logger.warning("photos obj=%s FAILED: %s\n URL: %s", obj_id, e, full_url)
|
||||
_log_failure(
|
||||
db,
|
||||
run_id,
|
||||
obj_id,
|
||||
"photos",
|
||||
full_url,
|
||||
_http_status_of(e),
|
||||
str(e),
|
||||
_body_of(e),
|
||||
)
|
||||
|
||||
if (i + 1) % 10 == 0:
|
||||
logger.info(
|
||||
"extras progress: %d/%d objs, %s",
|
||||
i + 1,
|
||||
len(all_objects),
|
||||
extras_counts,
|
||||
)
|
||||
# Periodic commit so failures are visible in admin UI even mid-run
|
||||
db.commit()
|
||||
logger.info("extras done: %s", extras_counts)
|
||||
|
||||
request_count = sess.request_count
|
||||
|
||||
# --- write all in one transaction ------------------------------------
|
||||
|
|
@ -517,6 +1059,7 @@ async def run_region_sweep(
|
|||
"flats": flat_count,
|
||||
"requests": request_count,
|
||||
"snapshot_date": snapshot_date.isoformat(),
|
||||
**extras_counts,
|
||||
}
|
||||
except Exception as e:
|
||||
logger.exception("kn sweep failed for region=%d", region_code)
|
||||
|
|
|
|||
|
|
@ -180,3 +180,22 @@ class BrowserSession:
|
|||
@property
|
||||
def request_count(self) -> int:
|
||||
return self._request_count
|
||||
|
||||
async def download_binary(self, url: str) -> bytes:
|
||||
"""Download a binary asset (e.g. photo PNG) reusing browser cookies/auth.
|
||||
|
||||
Uses Playwright APIRequest which goes through the browser context — same
|
||||
cookies, same TLS fingerprint as the page itself.
|
||||
"""
|
||||
if self._context is None:
|
||||
raise RuntimeError("BrowserSession not bootstrapped")
|
||||
await jitter_sleep(200, 500) # Lighter throttle for static assets.
|
||||
self._request_count += 1
|
||||
resp = await self._context.request.get(
|
||||
url,
|
||||
headers={"Authorization": self.auth} if self.auth else {},
|
||||
)
|
||||
if resp.status != 200:
|
||||
body = await resp.text()
|
||||
raise RuntimeError(f"binary http {resp.status}: {body[:200]}")
|
||||
return await resp.body()
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ def scrape_kn_region(
|
|||
region_code: int,
|
||||
developers: list[str] | None = None,
|
||||
skip_jitter: bool = False,
|
||||
extras: bool = True,
|
||||
download_photos: bool = False,
|
||||
fetch_flats: bool = True,
|
||||
) -> dict[str, Any]:
|
||||
"""Run a full kn-API sweep for one region. Returns summary dict.
|
||||
|
||||
|
|
@ -41,12 +44,20 @@ def scrape_kn_region(
|
|||
state_path = None
|
||||
|
||||
logger.info(
|
||||
"scrape_kn_region start region=%s devs=%s state=%s", region_code, developers, state_path
|
||||
"scrape_kn_region start region=%s devs=%s extras=%s photos=%s state=%s",
|
||||
region_code,
|
||||
developers,
|
||||
extras,
|
||||
download_photos,
|
||||
state_path,
|
||||
)
|
||||
return asyncio.run(
|
||||
run_region_sweep(
|
||||
region_code=region_code,
|
||||
developers=developers,
|
||||
load_state=state_path,
|
||||
fetch_flats=fetch_flats,
|
||||
extras=extras,
|
||||
download_photos_binary=download_photos,
|
||||
)
|
||||
)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from pathlib import Path
|
|||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "backend"))
|
||||
|
||||
from app.services.scrapers.domrf_kn import run_region_sweep # noqa: E402
|
||||
from app.services.scrapers.domrf_kn import probe_endpoint, run_region_sweep # noqa: E402
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
|
@ -27,8 +27,7 @@ def main() -> None:
|
|||
"--regions",
|
||||
type=int,
|
||||
nargs="+",
|
||||
required=True,
|
||||
help="Region codes (1..89). e.g. --regions 66 77",
|
||||
help="Region codes (1..89). e.g. --regions 66 77. Required unless --probe.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--developers",
|
||||
|
|
@ -36,23 +35,46 @@ def main() -> None:
|
|||
nargs="*",
|
||||
help="Optional developer IDs in '<id>_0' format. e.g. --developers 6208_0",
|
||||
)
|
||||
parser.add_argument("--headed", action="store_true", help="Show Chromium window during bootstrap")
|
||||
parser.add_argument(
|
||||
"--headed", action="store_true", help="Show Chromium window during bootstrap"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save-state",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to dump Playwright storage_state.json after bootstrap (commit to git for server reuse)",
|
||||
help="Path to dump Playwright storage_state.json after bootstrap",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--load-state",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to load Playwright storage_state.json (server uses fingerprint captured locally)",
|
||||
help="Path to load Playwright storage_state.json (server reuse)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--bootstrap-only",
|
||||
action="store_true",
|
||||
help="Just bootstrap a session and exit — useful for capturing storage_state without scraping",
|
||||
help="Just bootstrap a session and exit (useful for --save-state)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-extras",
|
||||
action="store_true",
|
||||
help="Skip sale_graph/sales_agg/infrastructure/photos endpoints",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--download-photos",
|
||||
action="store_true",
|
||||
help="Also download photo binaries to data/raw/domrf_photos/{obj_id}/",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-flats",
|
||||
action="store_true",
|
||||
help="Skip per-flat fetch (only objects + extras)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--probe",
|
||||
type=str,
|
||||
default=None,
|
||||
help="One-shot probe of a URL or path (e.g. '/сервисы/api/object/60160/sale_graph?type=apartments')",
|
||||
)
|
||||
parser.add_argument("--verbose", "-v", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
|
@ -66,10 +88,22 @@ def main() -> None:
|
|||
print("WARNING: DATABASE_URL not set; using default from app.core.config")
|
||||
|
||||
async def _run() -> None:
|
||||
if args.probe:
|
||||
result = await probe_endpoint(
|
||||
args.probe,
|
||||
region_code=(args.regions or [66])[0],
|
||||
headed=args.headed,
|
||||
load_state=args.load_state,
|
||||
)
|
||||
print(f"PROBE {result['full_url']}")
|
||||
print(f" status: {result['status']} ({result['content_type']})")
|
||||
print(f" body: {result['body_preview']}")
|
||||
return
|
||||
|
||||
if args.bootstrap_only:
|
||||
from app.services.scrapers.stealth import BrowserSession
|
||||
async with BrowserSession(
|
||||
region_code=args.regions[0],
|
||||
region_code=(args.regions or [66])[0],
|
||||
headed=args.headed,
|
||||
load_state=args.load_state,
|
||||
save_state=args.save_state,
|
||||
|
|
@ -77,6 +111,9 @@ def main() -> None:
|
|||
print(f"bootstrap-only: state saved to {args.save_state}")
|
||||
return
|
||||
|
||||
if not args.regions:
|
||||
parser.error("--regions is required unless --probe / --bootstrap-only")
|
||||
|
||||
for region in args.regions:
|
||||
summary = await run_region_sweep(
|
||||
region_code=region,
|
||||
|
|
@ -84,6 +121,9 @@ def main() -> None:
|
|||
headed=args.headed,
|
||||
load_state=args.load_state,
|
||||
save_state=args.save_state if region == args.regions[0] else None,
|
||||
fetch_flats=not args.no_flats,
|
||||
extras=not args.no_extras,
|
||||
download_photos_binary=args.download_photos,
|
||||
)
|
||||
print(f"region={region} -> {summary}")
|
||||
|
||||
|
|
|
|||
97
data/sql/51_schema_kn_extras.sql
Normal file
97
data/sql/51_schema_kn_extras.sql
Normal file
|
|
@ -0,0 +1,97 @@
|
|||
-- Extras-эндпойнты наш.дом.рф (sale_graph, sales_agg, infrastructure, photos)
|
||||
-- + журнал ошибок per-request для ручной проверки в браузере.
|
||||
-- См. план C:\Users\user\.claude\plans\crispy-waddling-swing.md.
|
||||
|
||||
-- ── Time-series продаж per-ЖК (apartments + parking) ──────────────────────
|
||||
-- Источник: GET /сервисы/api/object/{obj_id}/sale_graph?type=apartments|parking
|
||||
CREATE TABLE IF NOT EXISTS domrf_kn_sale_graph (
|
||||
obj_id BIGINT NOT NULL,
|
||||
report_month DATE NOT NULL, -- '01.04.2025' → 2025-04-01
|
||||
type TEXT NOT NULL, -- 'apartments' | 'parking'
|
||||
realised INT, -- штук реализовано
|
||||
contracted INT, -- штук законтрактовано (ДКП)
|
||||
area_sq NUMERIC, -- м² за месяц
|
||||
price_avg NUMERIC, -- средняя цена ₽/м²
|
||||
snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE,
|
||||
UNIQUE (obj_id, report_month, type, snapshot_date)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sg_obj
|
||||
ON domrf_kn_sale_graph (obj_id, snapshot_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_sg_month
|
||||
ON domrf_kn_sale_graph (report_month, type);
|
||||
|
||||
-- ── Текущие агрегаты per-ЖК (apartments + nonliv + parking) ───────────────
|
||||
-- Источник: GET /сервисы/api/object/{obj_id}/sales_agg
|
||||
CREATE TABLE IF NOT EXISTS domrf_kn_sales_agg (
|
||||
obj_id BIGINT NOT NULL,
|
||||
type TEXT NOT NULL, -- 'apartments'|'nonliv'|'parking'
|
||||
name TEXT, -- 'Квартира'|'Нежилое помещение'|'Машино-место'
|
||||
total INT, -- всего юнитов в ЖК
|
||||
realised INT, -- реализовано
|
||||
perc NUMERIC, -- sold % (server-calculated)
|
||||
snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE,
|
||||
UNIQUE (obj_id, type, snapshot_date)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_sa_obj
|
||||
ON domrf_kn_sales_agg (obj_id, snapshot_date);
|
||||
|
||||
-- ── POI вокруг ЖК (нормализованная) ───────────────────────────────────────
|
||||
-- Источник: GET /сервисы/api/object/{obj_id}/infrastructure
|
||||
CREATE TABLE IF NOT EXISTS domrf_kn_infrastructure (
|
||||
obj_id BIGINT NOT NULL,
|
||||
poi_name TEXT NOT NULL,
|
||||
poi_subtitle TEXT,
|
||||
poi_category TEXT, -- tabName: Развлечения|Образование|...
|
||||
poi_address TEXT,
|
||||
poi_lat NUMERIC,
|
||||
poi_lon NUMERIC,
|
||||
distance_m NUMERIC,
|
||||
snapshot_date DATE NOT NULL DEFAULT CURRENT_DATE,
|
||||
UNIQUE (obj_id, poi_name, poi_lat, poi_lon, snapshot_date)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_infra_obj
|
||||
ON domrf_kn_infrastructure (obj_id, snapshot_date);
|
||||
CREATE INDEX IF NOT EXISTS idx_infra_cat_dist
|
||||
ON domrf_kn_infrastructure (poi_category, distance_m);
|
||||
|
||||
-- ── Фото-метаданные (binary в data/raw/domrf_photos/{obj_id}/{file_id}.png) ─
|
||||
-- Источник: GET /сервисы/api/object/construction/progress/photo/{obj_id}
|
||||
CREATE TABLE IF NOT EXISTS domrf_kn_photos (
|
||||
obj_id BIGINT NOT NULL,
|
||||
obj_file_id TEXT NOT NULL, -- DOM.РФ хеш-ID, идемпотентен
|
||||
ord_num INT,
|
||||
photo_url TEXT, -- абсолютный URL DOM.РФ
|
||||
photo_dttm TIMESTAMP,
|
||||
period_dt DATE, -- '2026-04-01'
|
||||
size_bytes BIGINT,
|
||||
photo_name TEXT, -- 'PRINZIP_43_new.jpg'
|
||||
ready_desc TEXT, -- 'Строится'/'Сдан'
|
||||
build_type TEXT, -- 'Жилое'/'Нежилое'
|
||||
hidden BOOLEAN,
|
||||
local_path TEXT, -- если binary скачан
|
||||
downloaded_at TIMESTAMPTZ,
|
||||
UNIQUE (obj_id, obj_file_id)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_photos_obj
|
||||
ON domrf_kn_photos (obj_id, period_dt DESC);
|
||||
|
||||
-- ── Журнал per-request ошибок для ручной проверки в браузере ──────────────
|
||||
CREATE TABLE IF NOT EXISTS kn_scrape_failures (
|
||||
failure_id BIGSERIAL PRIMARY KEY,
|
||||
run_id BIGINT REFERENCES kn_scrape_runs(run_id) ON DELETE CASCADE,
|
||||
obj_id BIGINT,
|
||||
endpoint TEXT, -- 'sale_graph'|'sales_agg'|'infra'|'photos'
|
||||
full_url TEXT, -- полный URL для копирования в браузер
|
||||
http_status INT,
|
||||
error TEXT,
|
||||
body_preview TEXT, -- первые ~500 байт ответа
|
||||
occurred_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_kn_fail_run
|
||||
ON kn_scrape_failures (run_id, occurred_at DESC);
|
||||
|
||||
COMMENT ON TABLE domrf_kn_sale_graph IS 'Time-series продаж per-ЖК. /сервисы/api/object/{id}/sale_graph?type=apartments|parking';
|
||||
COMMENT ON TABLE domrf_kn_sales_agg IS 'Текущие агрегаты per-ЖК. /сервисы/api/object/{id}/sales_agg';
|
||||
COMMENT ON TABLE domrf_kn_infrastructure IS 'POI вокруг ЖК. /сервисы/api/object/{id}/infrastructure';
|
||||
COMMENT ON TABLE domrf_kn_photos IS 'Фото прогресса стройки. /сервисы/api/object/construction/progress/photo/{id}';
|
||||
COMMENT ON TABLE kn_scrape_failures IS 'Per-request ошибки парсера для ручной проверки full_url в браузере.';
|
||||
|
|
@ -25,6 +25,18 @@ interface TriggerResp {
|
|||
developers: string[] | null;
|
||||
}
|
||||
|
||||
interface FailureRow {
|
||||
failure_id: number;
|
||||
run_id: number | null;
|
||||
obj_id: number | null;
|
||||
endpoint: string | null;
|
||||
full_url: string | null;
|
||||
http_status: number | null;
|
||||
error: string | null;
|
||||
body_preview: string | null;
|
||||
occurred_at: string | null;
|
||||
}
|
||||
|
||||
export default function ScrapeAdminPage() {
|
||||
const [token, setToken] = useState<string>(() =>
|
||||
typeof window === "undefined"
|
||||
|
|
@ -34,6 +46,10 @@ export default function ScrapeAdminPage() {
|
|||
const [region, setRegion] = useState(66);
|
||||
const [devs, setDevs] = useState("");
|
||||
const [fetchFlats, setFetchFlats] = useState(true);
|
||||
const [extras, setExtras] = useState(true);
|
||||
const [downloadPhotos, setDownloadPhotos] = useState(false);
|
||||
const [showFailures, setShowFailures] = useState(false);
|
||||
const [failuresRunId, setFailuresRunId] = useState<number | "">("");
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const runs = useQuery({
|
||||
|
|
@ -60,12 +76,35 @@ export default function ScrapeAdminPage() {
|
|||
.filter(Boolean)
|
||||
: null,
|
||||
fetch_flats: fetchFlats,
|
||||
extras,
|
||||
download_photos: downloadPhotos,
|
||||
}),
|
||||
}),
|
||||
onSuccess: () =>
|
||||
queryClient.invalidateQueries({ queryKey: ["scrape-runs"] }),
|
||||
});
|
||||
|
||||
const failures = useQuery({
|
||||
queryKey: ["scrape-failures", failuresRunId, token],
|
||||
queryFn: () => {
|
||||
const q =
|
||||
failuresRunId !== ""
|
||||
? `?run_id=${failuresRunId}&limit=100`
|
||||
: "?limit=100";
|
||||
return apiFetch<FailureRow[]>(`/api/v1/admin/scrape/failures${q}`, {
|
||||
headers: { "X-Admin-Token": token },
|
||||
});
|
||||
},
|
||||
enabled: !!token && showFailures,
|
||||
refetchInterval: showFailures ? 5000 : false,
|
||||
});
|
||||
|
||||
const copyUrl = (url: string) => {
|
||||
if (typeof navigator !== "undefined" && navigator.clipboard) {
|
||||
navigator.clipboard.writeText(url);
|
||||
}
|
||||
};
|
||||
|
||||
const saveToken = (v: string) => {
|
||||
setToken(v);
|
||||
localStorage.setItem("admin_token", v);
|
||||
|
|
@ -124,7 +163,38 @@ export default function ScrapeAdminPage() {
|
|||
checked={fetchFlats}
|
||||
onChange={(e) => setFetchFlats(e.target.checked)}
|
||||
/>
|
||||
<span>Также собрать квартиры</span>
|
||||
<span>Квартиры</span>
|
||||
</label>
|
||||
<label
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
marginTop: 22,
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={extras}
|
||||
onChange={(e) => setExtras(e.target.checked)}
|
||||
/>
|
||||
<span>Extras (sale_graph + agg + infra + photos)</span>
|
||||
</label>
|
||||
<label
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
marginTop: 22,
|
||||
}}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={downloadPhotos}
|
||||
onChange={(e) => setDownloadPhotos(e.target.checked)}
|
||||
disabled={!extras}
|
||||
/>
|
||||
<span>Скачать фото (binary)</span>
|
||||
</label>
|
||||
</div>
|
||||
<div style={{ marginTop: 16 }}>
|
||||
|
|
@ -148,6 +218,127 @@ export default function ScrapeAdminPage() {
|
|||
</div>
|
||||
</section>
|
||||
|
||||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||
<div
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
justifyContent: "space-between",
|
||||
}}
|
||||
>
|
||||
<h2 style={{ margin: 0, fontSize: 16 }}>
|
||||
Журнал ошибок (per-request)
|
||||
</h2>
|
||||
<div style={{ display: "flex", gap: 8, alignItems: "center" }}>
|
||||
<input
|
||||
type="number"
|
||||
value={failuresRunId}
|
||||
onChange={(e) =>
|
||||
setFailuresRunId(
|
||||
e.target.value === "" ? "" : Number(e.target.value),
|
||||
)
|
||||
}
|
||||
placeholder="run_id (опц.)"
|
||||
style={{ ...inputStyle, width: 130 }}
|
||||
/>
|
||||
<button
|
||||
onClick={() => setShowFailures((v) => !v)}
|
||||
disabled={!token}
|
||||
style={secondaryBtn}
|
||||
>
|
||||
{showFailures ? "Скрыть" : "Показать"}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
{showFailures ? (
|
||||
failures.isLoading ? (
|
||||
<p style={{ color: "#5b6066" }}>Загрузка…</p>
|
||||
) : failures.isError ? (
|
||||
<p style={{ color: "#b3261e" }}>Ошибка: {String(failures.error)}</p>
|
||||
) : (failures.data?.length ?? 0) === 0 ? (
|
||||
<p style={{ color: "#5b6066" }}>Ошибок нет.</p>
|
||||
) : (
|
||||
<table
|
||||
style={{
|
||||
width: "100%",
|
||||
borderCollapse: "collapse",
|
||||
fontSize: 12,
|
||||
marginTop: 12,
|
||||
}}
|
||||
>
|
||||
<thead>
|
||||
<tr style={{ background: "#fef2f2" }}>
|
||||
{[
|
||||
"Когда",
|
||||
"Run",
|
||||
"obj",
|
||||
"endpoint",
|
||||
"Status",
|
||||
"URL",
|
||||
"Body",
|
||||
].map((h) => (
|
||||
<th key={h} style={th}>
|
||||
{h}
|
||||
</th>
|
||||
))}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{(failures.data ?? []).map((f) => (
|
||||
<tr
|
||||
key={f.failure_id}
|
||||
style={{ borderBottom: "1px solid #eef0f3" }}
|
||||
>
|
||||
<td style={td}>
|
||||
{f.occurred_at?.slice(0, 19).replace("T", " ")}
|
||||
</td>
|
||||
<td style={td}>{f.run_id ?? "—"}</td>
|
||||
<td style={td}>{f.obj_id ?? "—"}</td>
|
||||
<td style={td}>{f.endpoint ?? "—"}</td>
|
||||
<td style={{ ...td, color: "#b3261e" }}>
|
||||
{f.http_status ?? "—"}
|
||||
</td>
|
||||
<td
|
||||
style={{ ...td, maxWidth: 380, fontFamily: "monospace" }}
|
||||
>
|
||||
{f.full_url ? (
|
||||
<button
|
||||
onClick={() => copyUrl(f.full_url!)}
|
||||
title="Копировать URL для проверки в браузере"
|
||||
style={{
|
||||
...secondaryBtn,
|
||||
padding: "2px 8px",
|
||||
fontSize: 11,
|
||||
cursor: "copy",
|
||||
}}
|
||||
>
|
||||
📋 {f.full_url.slice(0, 60)}…
|
||||
</button>
|
||||
) : (
|
||||
"—"
|
||||
)}
|
||||
</td>
|
||||
<td
|
||||
style={{
|
||||
...td,
|
||||
fontFamily: "monospace",
|
||||
fontSize: 11,
|
||||
color: "#5b6066",
|
||||
maxWidth: 240,
|
||||
}}
|
||||
>
|
||||
{f.body_preview?.slice(0, 80) ??
|
||||
f.error?.slice(0, 80) ??
|
||||
""}
|
||||
</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
)
|
||||
) : null}
|
||||
</section>
|
||||
|
||||
<section style={{ ...cardStyle, marginTop: 16 }}>
|
||||
<h2 style={{ marginTop: 0, fontSize: 16 }}>Последние прогоны</h2>
|
||||
{!token ? (
|
||||
|
|
@ -264,6 +455,15 @@ const primaryBtn = {
|
|||
fontWeight: 600,
|
||||
cursor: "pointer",
|
||||
};
|
||||
const secondaryBtn = {
|
||||
padding: "8px 14px",
|
||||
background: "#f3f4f6",
|
||||
color: "#1f2937",
|
||||
border: "1px solid #d1d5db",
|
||||
borderRadius: 6,
|
||||
fontSize: 13,
|
||||
cursor: "pointer",
|
||||
};
|
||||
const th = {
|
||||
padding: "8px 10px",
|
||||
textAlign: "left" as const,
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
Loading…
Add table
Reference in a new issue