fix ui lag
This commit is contained in:
parent
c079ac2a77
commit
b3c15bb8ea
17 changed files with 473 additions and 72 deletions
|
|
@ -69,17 +69,65 @@ def trigger_kn_sweep(
|
||||||
|
|
||||||
@router.get("/queue")
|
@router.get("/queue")
|
||||||
def queue_status(
|
def queue_status(
|
||||||
|
db: Annotated[Session, Depends(get_db)],
|
||||||
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||||
) -> dict[str, Any]:
|
) -> dict[str, Any]:
|
||||||
"""Snapshot of Celery worker(s) state: active tasks, reserved (queued on
|
"""Snapshot of Celery state, optimised for UI poll latency.
|
||||||
worker, not yet started), scheduled (with ETA), beat-schedule, and queue
|
|
||||||
depth in Redis. If no worker is running, all worker-side fields are empty.
|
- active: from kn_scrape_runs WHERE status='running' (instant, richer info
|
||||||
|
than celery inspect — includes objects_count/flats_count/requests_count).
|
||||||
|
- reserved + workers: celery inspect with short timeout (500 ms), called in
|
||||||
|
a thread so a missing worker can't block the request.
|
||||||
|
- queue_depth: Redis LLEN on default queue (single round-trip).
|
||||||
|
- scheduled: omitted — we don't use ETA tasks.
|
||||||
|
|
||||||
|
Worst-case latency ≈ 600 ms even if no worker is reachable, vs ~8 s with
|
||||||
|
the previous 4×2 s blocking inspect calls.
|
||||||
"""
|
"""
|
||||||
_check_token(x_admin_token)
|
_check_token(x_admin_token)
|
||||||
|
|
||||||
|
import concurrent.futures
|
||||||
|
import time
|
||||||
|
|
||||||
from app.workers.celery_app import celery_app
|
from app.workers.celery_app import celery_app
|
||||||
|
|
||||||
inspect = celery_app.control.inspect(timeout=2.0)
|
# 1) Active runs straight from DB — no broker round-trip.
|
||||||
|
active_rows = (
|
||||||
|
db.execute(
|
||||||
|
text(
|
||||||
|
"""
|
||||||
|
SELECT run_id, started_at, region_codes, developer_ids,
|
||||||
|
objects_count, flats_count, requests_count
|
||||||
|
FROM kn_scrape_runs
|
||||||
|
WHERE status = 'running'
|
||||||
|
ORDER BY started_at DESC
|
||||||
|
LIMIT 20
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
)
|
||||||
|
.mappings()
|
||||||
|
.all()
|
||||||
|
)
|
||||||
|
active: list[dict[str, Any]] = [
|
||||||
|
{
|
||||||
|
"worker": "db",
|
||||||
|
"task_id": f"run-{r['run_id']}",
|
||||||
|
"name": "scrape_kn_region",
|
||||||
|
"args": [list(r["region_codes"] or []), list(r["developer_ids"] or [])],
|
||||||
|
"kwargs": {
|
||||||
|
"objects": r["objects_count"],
|
||||||
|
"flats": r["flats_count"],
|
||||||
|
"requests": r["requests_count"],
|
||||||
|
},
|
||||||
|
"time_start": (int(r["started_at"].timestamp()) if r["started_at"] else None),
|
||||||
|
"eta": None,
|
||||||
|
}
|
||||||
|
for r in active_rows
|
||||||
|
]
|
||||||
|
|
||||||
|
# 2) Run reserved + ping in parallel with a tight timeout so the endpoint
|
||||||
|
# never hangs when the broker is unreachable or worker is gone.
|
||||||
|
inspect = celery_app.control.inspect(timeout=0.5)
|
||||||
|
|
||||||
def _flatten(per_worker: dict[str, list[dict[str, Any]]] | None) -> list[dict[str, Any]]:
|
def _flatten(per_worker: dict[str, list[dict[str, Any]]] | None) -> list[dict[str, Any]]:
|
||||||
if not per_worker:
|
if not per_worker:
|
||||||
|
|
@ -100,42 +148,43 @@ def queue_status(
|
||||||
)
|
)
|
||||||
return out
|
return out
|
||||||
|
|
||||||
try:
|
def _safe(fn):
|
||||||
active = _flatten(inspect.active())
|
try:
|
||||||
except Exception:
|
return fn()
|
||||||
active = []
|
except Exception:
|
||||||
try:
|
return None
|
||||||
reserved = _flatten(inspect.reserved())
|
|
||||||
except Exception:
|
|
||||||
reserved = []
|
|
||||||
try:
|
|
||||||
scheduled = _flatten(inspect.scheduled())
|
|
||||||
except Exception:
|
|
||||||
scheduled = []
|
|
||||||
|
|
||||||
# Pending in broker queue (not yet picked up by any worker).
|
deadline = time.monotonic() + 0.8
|
||||||
|
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex:
|
||||||
|
f_reserved = ex.submit(_safe, inspect.reserved)
|
||||||
|
f_ping = ex.submit(_safe, inspect.ping)
|
||||||
|
try:
|
||||||
|
reserved_raw = f_reserved.result(timeout=max(0.1, deadline - time.monotonic()))
|
||||||
|
except concurrent.futures.TimeoutError:
|
||||||
|
reserved_raw = None
|
||||||
|
try:
|
||||||
|
ping_resp = f_ping.result(timeout=max(0.1, deadline - time.monotonic()))
|
||||||
|
except concurrent.futures.TimeoutError:
|
||||||
|
ping_resp = None
|
||||||
|
|
||||||
|
reserved = _flatten(reserved_raw)
|
||||||
|
workers = list((ping_resp or {}).keys())
|
||||||
|
|
||||||
|
# 3) Pending in broker queue (not yet picked up by any worker).
|
||||||
queue_depth: int | None = None
|
queue_depth: int | None = None
|
||||||
try:
|
try:
|
||||||
with celery_app.connection_or_acquire() as conn:
|
with celery_app.connection_or_acquire() as conn:
|
||||||
with conn.channel() as channel:
|
with conn.channel() as channel:
|
||||||
# Default queue name is 'celery'.
|
|
||||||
queue_depth = channel.client.llen("celery")
|
queue_depth = channel.client.llen("celery")
|
||||||
except Exception:
|
except Exception:
|
||||||
queue_depth = None
|
queue_depth = None
|
||||||
|
|
||||||
workers: list[str] = []
|
|
||||||
try:
|
|
||||||
ping_resp = inspect.ping() or {}
|
|
||||||
workers = list(ping_resp.keys())
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
"workers": workers,
|
"workers": workers,
|
||||||
"queue_depth": queue_depth,
|
"queue_depth": queue_depth,
|
||||||
"active": active,
|
"active": active,
|
||||||
"reserved": reserved,
|
"reserved": reserved,
|
||||||
"scheduled": scheduled,
|
"scheduled": [],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
||||||
87
backend/app/api/v1/photos.py
Normal file
87
backend/app/api/v1/photos.py
Normal file
|
|
@ -0,0 +1,87 @@
|
||||||
|
"""Serve cached DOM.РФ photos.
|
||||||
|
|
||||||
|
GET /api/v1/photos/{obj_id}/{file_id}?size=thumb|full
|
||||||
|
|
||||||
|
- size=thumb (default): WebP 320×240 from local cache. If thumb is missing
|
||||||
|
but the original is present locally, generate one on the fly. If neither
|
||||||
|
is present, redirect to the upstream DOM.РФ URL.
|
||||||
|
- size=full: original PNG/JPG if cached locally, else redirect upstream.
|
||||||
|
|
||||||
|
This isolates the frontend grid from upstream latency and from Next.js dev-mode
|
||||||
|
optimizer cold-hit cost.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from typing import Annotated, Literal
|
||||||
|
|
||||||
|
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||||
|
from fastapi.responses import FileResponse, RedirectResponse, Response
|
||||||
|
from sqlalchemy import text
|
||||||
|
from sqlalchemy.orm import Session
|
||||||
|
|
||||||
|
from app.core.db import get_db
|
||||||
|
from app.services.photos.thumbs import make_thumbnail
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
router = APIRouter()
|
||||||
|
|
||||||
|
|
||||||
|
_PHOTO_LOOKUP_SQL = text(
|
||||||
|
"""
|
||||||
|
SELECT local_path, thumb_path, photo_url
|
||||||
|
FROM domrf_kn_photos
|
||||||
|
WHERE obj_id = :obj AND obj_file_id = :fid
|
||||||
|
LIMIT 1
|
||||||
|
"""
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@router.get("/{obj_id}/{file_id}")
|
||||||
|
def get_photo(
|
||||||
|
db: Annotated[Session, Depends(get_db)],
|
||||||
|
obj_id: int,
|
||||||
|
file_id: str,
|
||||||
|
size: Annotated[Literal["thumb", "full"], Query()] = "thumb",
|
||||||
|
) -> Response:
|
||||||
|
row = db.execute(_PHOTO_LOOKUP_SQL, {"obj": obj_id, "fid": file_id}).mappings().first()
|
||||||
|
if not row:
|
||||||
|
raise HTTPException(status_code=404, detail="photo not registered")
|
||||||
|
|
||||||
|
local_path = row["local_path"]
|
||||||
|
thumb_path = row["thumb_path"]
|
||||||
|
upstream = row["photo_url"]
|
||||||
|
|
||||||
|
headers = {"Cache-Control": "public, max-age=604800, immutable"}
|
||||||
|
|
||||||
|
if size == "full":
|
||||||
|
if local_path and Path(local_path).exists():
|
||||||
|
return FileResponse(local_path, headers=headers)
|
||||||
|
if upstream:
|
||||||
|
return RedirectResponse(upstream, status_code=302)
|
||||||
|
raise HTTPException(status_code=404, detail="full photo unavailable")
|
||||||
|
|
||||||
|
# size=thumb
|
||||||
|
if thumb_path and Path(thumb_path).exists():
|
||||||
|
return FileResponse(thumb_path, media_type="image/webp", headers=headers)
|
||||||
|
|
||||||
|
# Generate thumb on the fly from cached original
|
||||||
|
if local_path and Path(local_path).exists():
|
||||||
|
generated = make_thumbnail(Path(local_path))
|
||||||
|
if generated and generated.exists():
|
||||||
|
db.execute(
|
||||||
|
text(
|
||||||
|
"UPDATE domrf_kn_photos SET thumb_path = :tp"
|
||||||
|
" WHERE obj_id = :o AND obj_file_id = :f"
|
||||||
|
),
|
||||||
|
{"tp": str(generated), "o": obj_id, "f": file_id},
|
||||||
|
)
|
||||||
|
db.commit()
|
||||||
|
return FileResponse(str(generated), media_type="image/webp", headers=headers)
|
||||||
|
|
||||||
|
# Last resort — redirect to upstream so the page still renders
|
||||||
|
if upstream:
|
||||||
|
return RedirectResponse(upstream, status_code=302)
|
||||||
|
raise HTTPException(status_code=404, detail="photo unavailable")
|
||||||
|
|
@ -5,7 +5,7 @@ import sentry_sdk
|
||||||
from fastapi import FastAPI
|
from fastapi import FastAPI
|
||||||
from fastapi.middleware.cors import CORSMiddleware
|
from fastapi.middleware.cors import CORSMiddleware
|
||||||
|
|
||||||
from app.api.v1 import admin_leads, admin_scrape, analytics, concepts, parcels
|
from app.api.v1 import admin_leads, admin_scrape, analytics, concepts, parcels, photos
|
||||||
from app.core.config import settings
|
from app.core.config import settings
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -35,6 +35,7 @@ app.include_router(parcels.router, prefix="/api/v1/parcels", tags=["parcels"])
|
||||||
app.include_router(analytics.router, prefix="/api/v1/analytics", tags=["analytics"])
|
app.include_router(analytics.router, prefix="/api/v1/analytics", tags=["analytics"])
|
||||||
app.include_router(admin_scrape.router, prefix="/api/v1/admin/scrape", tags=["admin"])
|
app.include_router(admin_scrape.router, prefix="/api/v1/admin/scrape", tags=["admin"])
|
||||||
app.include_router(admin_leads.router, prefix="/api/v1/admin/leads", tags=["admin"])
|
app.include_router(admin_leads.router, prefix="/api/v1/admin/leads", tags=["admin"])
|
||||||
|
app.include_router(photos.router, prefix="/api/v1/photos", tags=["photos"])
|
||||||
|
|
||||||
|
|
||||||
@app.get("/health")
|
@app.get("/health")
|
||||||
|
|
|
||||||
|
|
@ -787,6 +787,10 @@ def object_photos(db: Session, obj_id: int, limit: int = 100) -> list[dict[str,
|
||||||
"obj_file_id": r["obj_file_id"],
|
"obj_file_id": r["obj_file_id"],
|
||||||
"ord_num": r["ord_num"],
|
"ord_num": r["ord_num"],
|
||||||
"photo_url": r["photo_url"],
|
"photo_url": r["photo_url"],
|
||||||
|
# Always serve thumbs through our backend — cached WebP, no upstream
|
||||||
|
# latency, no Next.js dev-mode optimizer cold-hit cost.
|
||||||
|
"thumb_url": f"/api/v1/photos/{obj_id}/{r['obj_file_id']}?size=thumb",
|
||||||
|
"full_url": f"/api/v1/photos/{obj_id}/{r['obj_file_id']}?size=full",
|
||||||
"photo_dttm": r["photo_dttm"].isoformat() if r["photo_dttm"] else None,
|
"photo_dttm": r["photo_dttm"].isoformat() if r["photo_dttm"] else None,
|
||||||
"period_dt": r["period_dt"].isoformat() if r["period_dt"] else None,
|
"period_dt": r["period_dt"].isoformat() if r["period_dt"] else None,
|
||||||
"size_bytes": r["size_bytes"],
|
"size_bytes": r["size_bytes"],
|
||||||
|
|
|
||||||
0
backend/app/services/photos/__init__.py
Normal file
0
backend/app/services/photos/__init__.py
Normal file
57
backend/app/services/photos/thumbs.py
Normal file
57
backend/app/services/photos/thumbs.py
Normal file
|
|
@ -0,0 +1,57 @@
|
||||||
|
"""Thumbnail generation for DOM.РФ photos.
|
||||||
|
|
||||||
|
Strategy: original PNG/JPG ~1-3 MB downloaded by scraper to
|
||||||
|
data/raw/domrf_photos/<obj_id>/<file_id>.{png,jpg}. We additionally write a
|
||||||
|
WebP cover-thumbnail at 320×240 (~10-30 KB) next to it, suffix `_thumb.webp`.
|
||||||
|
|
||||||
|
The frontend grid uses /api/v1/photos/<obj_id>/<file_id>?size=thumb. Lightbox
|
||||||
|
opens the original DOM.РФ URL (we don't need to mirror originals).
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from PIL import Image, ImageOps
|
||||||
|
|
||||||
|
logger = logging.getLogger(__name__)
|
||||||
|
|
||||||
|
THUMB_SIZE: tuple[int, int] = (320, 240)
|
||||||
|
THUMB_QUALITY: int = 70
|
||||||
|
THUMB_SUFFIX: str = "_thumb.webp"
|
||||||
|
|
||||||
|
|
||||||
|
def thumb_path_for(src: Path) -> Path:
|
||||||
|
"""Return canonical thumbnail path next to the source file."""
|
||||||
|
return src.with_name(src.stem + THUMB_SUFFIX)
|
||||||
|
|
||||||
|
|
||||||
|
def make_thumbnail(
|
||||||
|
src: Path,
|
||||||
|
*,
|
||||||
|
size: tuple[int, int] = THUMB_SIZE,
|
||||||
|
quality: int = THUMB_QUALITY,
|
||||||
|
overwrite: bool = False,
|
||||||
|
) -> Path | None:
|
||||||
|
"""Generate a WebP cover-thumbnail next to src. Returns thumb path on success.
|
||||||
|
|
||||||
|
Skips if thumb already exists and overwrite=False. Returns None on any error
|
||||||
|
(logged) so callers can keep going with the next file.
|
||||||
|
"""
|
||||||
|
if not src.exists():
|
||||||
|
return None
|
||||||
|
dst = thumb_path_for(src)
|
||||||
|
if dst.exists() and not overwrite:
|
||||||
|
return dst
|
||||||
|
try:
|
||||||
|
with Image.open(src) as im:
|
||||||
|
im = ImageOps.exif_transpose(im)
|
||||||
|
im = im.convert("RGB")
|
||||||
|
im = ImageOps.fit(im, size, method=Image.Resampling.LANCZOS)
|
||||||
|
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
im.save(dst, format="WEBP", quality=quality, method=4)
|
||||||
|
return dst
|
||||||
|
except Exception as e:
|
||||||
|
logger.warning("thumbnail %s failed: %s", src, e)
|
||||||
|
return None
|
||||||
|
|
@ -429,11 +429,11 @@ UPSERT_PHOTO_SQL = text(
|
||||||
INSERT INTO domrf_kn_photos (
|
INSERT INTO domrf_kn_photos (
|
||||||
obj_id, obj_file_id, ord_num, photo_url, photo_dttm, period_dt,
|
obj_id, obj_file_id, ord_num, photo_url, photo_dttm, period_dt,
|
||||||
size_bytes, photo_name, ready_desc, build_type, hidden,
|
size_bytes, photo_name, ready_desc, build_type, hidden,
|
||||||
local_path, downloaded_at
|
local_path, thumb_path, downloaded_at
|
||||||
) VALUES (
|
) VALUES (
|
||||||
:obj_id, :obj_file_id, :ord_num, :photo_url, :photo_dttm, :period_dt,
|
:obj_id, :obj_file_id, :ord_num, :photo_url, :photo_dttm, :period_dt,
|
||||||
:size_bytes, :photo_name, :ready_desc, :build_type, :hidden,
|
:size_bytes, :photo_name, :ready_desc, :build_type, :hidden,
|
||||||
:local_path, :downloaded_at
|
:local_path, :thumb_path, :downloaded_at
|
||||||
)
|
)
|
||||||
ON CONFLICT (obj_id, obj_file_id) DO UPDATE SET
|
ON CONFLICT (obj_id, obj_file_id) DO UPDATE SET
|
||||||
ord_num = EXCLUDED.ord_num,
|
ord_num = EXCLUDED.ord_num,
|
||||||
|
|
@ -445,6 +445,7 @@ UPSERT_PHOTO_SQL = text(
|
||||||
ready_desc = EXCLUDED.ready_desc,
|
ready_desc = EXCLUDED.ready_desc,
|
||||||
hidden = EXCLUDED.hidden,
|
hidden = EXCLUDED.hidden,
|
||||||
local_path = COALESCE(EXCLUDED.local_path, domrf_kn_photos.local_path),
|
local_path = COALESCE(EXCLUDED.local_path, domrf_kn_photos.local_path),
|
||||||
|
thumb_path = COALESCE(EXCLUDED.thumb_path, domrf_kn_photos.thumb_path),
|
||||||
downloaded_at = COALESCE(EXCLUDED.downloaded_at, domrf_kn_photos.downloaded_at)
|
downloaded_at = COALESCE(EXCLUDED.downloaded_at, domrf_kn_photos.downloaded_at)
|
||||||
"""
|
"""
|
||||||
)
|
)
|
||||||
|
|
@ -699,9 +700,11 @@ def upsert_photos(
|
||||||
obj_id: int,
|
obj_id: int,
|
||||||
photos: list[dict[str, Any]],
|
photos: list[dict[str, Any]],
|
||||||
local_paths: dict[str, str] | None = None,
|
local_paths: dict[str, str] | None = None,
|
||||||
|
thumb_paths: dict[str, str] | None = None,
|
||||||
) -> int:
|
) -> int:
|
||||||
"""Upsert photo metadata. local_paths maps file_id → relative path on disk."""
|
"""Upsert photo metadata. local_paths/thumb_paths map file_id → path."""
|
||||||
local_paths = local_paths or {}
|
local_paths = local_paths or {}
|
||||||
|
thumb_paths = thumb_paths or {}
|
||||||
now_dt = datetime.now()
|
now_dt = datetime.now()
|
||||||
n = 0
|
n = 0
|
||||||
for p in photos:
|
for p in photos:
|
||||||
|
|
@ -709,6 +712,7 @@ def upsert_photos(
|
||||||
if not file_id:
|
if not file_id:
|
||||||
continue
|
continue
|
||||||
local = local_paths.get(file_id)
|
local = local_paths.get(file_id)
|
||||||
|
thumb = thumb_paths.get(file_id)
|
||||||
try:
|
try:
|
||||||
db.execute(
|
db.execute(
|
||||||
UPSERT_PHOTO_SQL,
|
UPSERT_PHOTO_SQL,
|
||||||
|
|
@ -725,6 +729,7 @@ def upsert_photos(
|
||||||
"build_type": p.get("objBuildTypeShortDesc"),
|
"build_type": p.get("objBuildTypeShortDesc"),
|
||||||
"hidden": _to_bool(p.get("objPhotoHiddenFlg")),
|
"hidden": _to_bool(p.get("objPhotoHiddenFlg")),
|
||||||
"local_path": local,
|
"local_path": local,
|
||||||
|
"thumb_path": thumb,
|
||||||
"downloaded_at": now_dt if local else None,
|
"downloaded_at": now_dt if local else None,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
@ -740,12 +745,18 @@ async def download_photos(
|
||||||
obj_id: int,
|
obj_id: int,
|
||||||
photos: list[dict[str, Any]],
|
photos: list[dict[str, Any]],
|
||||||
base_dir: Path,
|
base_dir: Path,
|
||||||
) -> dict[str, str]:
|
) -> tuple[dict[str, str], dict[str, str]]:
|
||||||
"""Download photo binaries to base_dir/{obj_id}/{file_id}.png (or .jpg).
|
"""Download photo binaries to base_dir/{obj_id}/{file_id}.png (or .jpg) +
|
||||||
Returns mapping {file_id: relative_path}. Skips if file already exists with
|
generate WebP thumbnails next to them.
|
||||||
matching size_bytes.
|
|
||||||
|
Returns a pair of mappings: (local_paths, thumb_paths) — both keyed by
|
||||||
|
file_id, value = relative path on disk. Skips originals already present
|
||||||
|
with matching size_bytes; thumbs skipped if already exist.
|
||||||
"""
|
"""
|
||||||
out: dict[str, str] = {}
|
from app.services.photos.thumbs import make_thumbnail, thumb_path_for
|
||||||
|
|
||||||
|
locals_out: dict[str, str] = {}
|
||||||
|
thumbs_out: dict[str, str] = {}
|
||||||
obj_dir = base_dir / str(obj_id)
|
obj_dir = base_dir / str(obj_id)
|
||||||
obj_dir.mkdir(parents=True, exist_ok=True)
|
obj_dir.mkdir(parents=True, exist_ok=True)
|
||||||
for p in photos:
|
for p in photos:
|
||||||
|
|
@ -760,15 +771,24 @@ async def download_photos(
|
||||||
# Skip if size matches
|
# Skip if size matches
|
||||||
expected_size = p.get("objSize")
|
expected_size = p.get("objSize")
|
||||||
if local.exists() and expected_size and local.stat().st_size == expected_size:
|
if local.exists() and expected_size and local.stat().st_size == expected_size:
|
||||||
out[file_id] = str(local)
|
locals_out[file_id] = str(local)
|
||||||
continue
|
else:
|
||||||
try:
|
try:
|
||||||
data = await sess.download_binary(url)
|
data = await sess.download_binary(url)
|
||||||
local.write_bytes(data)
|
local.write_bytes(data)
|
||||||
out[file_id] = str(local)
|
locals_out[file_id] = str(local)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("download photo obj=%s file=%s failed: %s", obj_id, file_id, e)
|
logger.warning("download photo obj=%s file=%s failed: %s", obj_id, file_id, e)
|
||||||
return out
|
continue
|
||||||
|
# Always (re)check thumbnail. make_thumbnail is no-op if it already exists.
|
||||||
|
existing_thumb = thumb_path_for(local)
|
||||||
|
if existing_thumb.exists():
|
||||||
|
thumbs_out[file_id] = str(existing_thumb)
|
||||||
|
else:
|
||||||
|
t = make_thumbnail(local)
|
||||||
|
if t:
|
||||||
|
thumbs_out[file_id] = str(t)
|
||||||
|
return locals_out, thumbs_out
|
||||||
|
|
||||||
|
|
||||||
# ── orchestrator ─────────────────────────────────────────────────────────────
|
# ── orchestrator ─────────────────────────────────────────────────────────────
|
||||||
|
|
@ -999,11 +1019,14 @@ async def run_region_sweep(
|
||||||
try:
|
try:
|
||||||
photos, full_url = await fetch_photos(sess, obj_id)
|
photos, full_url = await fetch_photos(sess, obj_id)
|
||||||
local_paths: dict[str, str] = {}
|
local_paths: dict[str, str] = {}
|
||||||
|
thumb_paths: dict[str, str] = {}
|
||||||
if download_photos_binary and photos:
|
if download_photos_binary and photos:
|
||||||
local_paths = await download_photos(sess, obj_id, photos, pdir)
|
local_paths, thumb_paths = await download_photos(
|
||||||
|
sess, obj_id, photos, pdir
|
||||||
|
)
|
||||||
extras_counts["photos_downloaded"] += len(local_paths)
|
extras_counts["photos_downloaded"] += len(local_paths)
|
||||||
extras_counts["photos_rows"] += upsert_photos(
|
extras_counts["photos_rows"] += upsert_photos(
|
||||||
db, obj_id, photos, local_paths
|
db, obj_id, photos, local_paths, thumb_paths
|
||||||
)
|
)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}"
|
full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}"
|
||||||
|
|
|
||||||
|
|
@ -20,6 +20,7 @@ dependencies = [
|
||||||
"celery>=5.4.0",
|
"celery>=5.4.0",
|
||||||
"playwright>=1.45.0",
|
"playwright>=1.45.0",
|
||||||
"tenacity>=9.0.0",
|
"tenacity>=9.0.0",
|
||||||
|
"pillow>=10.4.0",
|
||||||
"weasyprint>=62.0",
|
"weasyprint>=62.0",
|
||||||
"ezdxf>=1.3.0",
|
"ezdxf>=1.3.0",
|
||||||
"openpyxl>=3.1.0",
|
"openpyxl>=3.1.0",
|
||||||
|
|
|
||||||
10
data/sql/53_schema_kn_thumbs.sql
Normal file
10
data/sql/53_schema_kn_thumbs.sql
Normal file
|
|
@ -0,0 +1,10 @@
|
||||||
|
-- Add thumbnail path column for cached locally-served thumbnails.
|
||||||
|
-- Generated by Pillow at download time (320×240 cover, WebP q=70).
|
||||||
|
-- Idempotent.
|
||||||
|
|
||||||
|
ALTER TABLE domrf_kn_photos
|
||||||
|
ADD COLUMN IF NOT EXISTS thumb_path TEXT;
|
||||||
|
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_kn_photos_obj_visible
|
||||||
|
ON domrf_kn_photos (obj_id)
|
||||||
|
WHERE COALESCE(hidden, FALSE) = FALSE;
|
||||||
|
|
@ -388,6 +388,24 @@ export default function ScrapeAdminPage() {
|
||||||
</p>
|
</p>
|
||||||
) : queue.data ? (
|
) : queue.data ? (
|
||||||
<>
|
<>
|
||||||
|
{queue.data.active.length === 0 &&
|
||||||
|
queue.data.reserved.length === 0 &&
|
||||||
|
queue.data.scheduled.length === 0 ? (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
padding: 12,
|
||||||
|
background: "#f0fdf4",
|
||||||
|
border: "1px solid #bbf7d0",
|
||||||
|
borderRadius: 6,
|
||||||
|
color: "#065f46",
|
||||||
|
fontSize: 13,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
✅ Worker онлайн ({queue.data.workers.join(", ")}) и ждёт задач.
|
||||||
|
Активных нет, очередь пуста. Нажми «Запустить сейчас» выше или
|
||||||
|
подожди расписание (cron <code>SCRAPE_KN_CRON</code>).
|
||||||
|
</div>
|
||||||
|
) : null}
|
||||||
{queue.data.active.length > 0 ? (
|
{queue.data.active.length > 0 ? (
|
||||||
<>
|
<>
|
||||||
<h3
|
<h3
|
||||||
|
|
|
||||||
|
|
@ -1,15 +1,66 @@
|
||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import Link from "next/link";
|
import Link from "next/link";
|
||||||
|
import dynamic from "next/dynamic";
|
||||||
import { useParams } from "next/navigation";
|
import { useParams } from "next/navigation";
|
||||||
|
|
||||||
import { KpiCard } from "@/components/analytics/KpiCard";
|
import { KpiCard } from "@/components/analytics/KpiCard";
|
||||||
import { ObjectInfraMap } from "@/components/analytics/ObjectInfraMap";
|
import { LazyMount } from "@/components/analytics/LazyMount";
|
||||||
import { ObjectPhotoGallery } from "@/components/analytics/ObjectPhotoGallery";
|
|
||||||
import { ObjectSaleChart } from "@/components/analytics/ObjectSaleChart";
|
|
||||||
import { Section } from "@/components/analytics/Section";
|
import { Section } from "@/components/analytics/Section";
|
||||||
import { useObjectDetail, useObjectSalesAgg } from "@/lib/analytics-api";
|
import { useObjectDetail, useObjectSalesAgg } from "@/lib/analytics-api";
|
||||||
|
|
||||||
|
// Heavy sections deferred so initial paint and main-thread are not blocked.
|
||||||
|
const ObjectSaleChart = dynamic(
|
||||||
|
() =>
|
||||||
|
import("@/components/analytics/ObjectSaleChart").then(
|
||||||
|
(m) => m.ObjectSaleChart,
|
||||||
|
),
|
||||||
|
{
|
||||||
|
ssr: false,
|
||||||
|
loading: () => (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
height: 300,
|
||||||
|
background: "#fafbfc",
|
||||||
|
border: "1px solid #eef0f3",
|
||||||
|
borderRadius: 8,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const ObjectInfraMap = dynamic(
|
||||||
|
() =>
|
||||||
|
import("@/components/analytics/ObjectInfraMap").then(
|
||||||
|
(m) => m.ObjectInfraMap,
|
||||||
|
),
|
||||||
|
{
|
||||||
|
ssr: false,
|
||||||
|
loading: () => (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
height: 360,
|
||||||
|
background: "#fafbfc",
|
||||||
|
border: "1px solid #eef0f3",
|
||||||
|
borderRadius: 8,
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const ObjectPhotoGallery = dynamic(
|
||||||
|
() =>
|
||||||
|
import("@/components/analytics/ObjectPhotoGallery").then(
|
||||||
|
(m) => m.ObjectPhotoGallery,
|
||||||
|
),
|
||||||
|
{
|
||||||
|
ssr: false,
|
||||||
|
loading: () => <div style={{ height: 320 }} />,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
export default function ObjectDrillInPage() {
|
export default function ObjectDrillInPage() {
|
||||||
const params = useParams<{ id: string }>();
|
const params = useParams<{ id: string }>();
|
||||||
const objId = params.id;
|
const objId = params.id;
|
||||||
|
|
@ -120,25 +171,31 @@ export default function ObjectDrillInPage() {
|
||||||
title="Динамика продаж"
|
title="Динамика продаж"
|
||||||
subtitle="DOM.РФ /sale_graph: реализовано (бары) + средняя цена ₽/м² (линия). Apartments + parking."
|
subtitle="DOM.РФ /sale_graph: реализовано (бары) + средняя цена ₽/м² (линия). Apartments + parking."
|
||||||
>
|
>
|
||||||
<ObjectSaleChart objId={objId} />
|
<LazyMount eager minHeight={300}>
|
||||||
|
<ObjectSaleChart objId={objId} />
|
||||||
|
</LazyMount>
|
||||||
</Section>
|
</Section>
|
||||||
|
|
||||||
<Section
|
<Section
|
||||||
title="Инфраструктура (POI вокруг ЖК)"
|
title="Инфраструктура (POI вокруг ЖК)"
|
||||||
subtitle="Из DOM.РФ /infrastructure. Категории кликабельны для фильтра."
|
subtitle="Из DOM.РФ /infrastructure. Категории кликабельны для фильтра."
|
||||||
>
|
>
|
||||||
<ObjectInfraMap
|
<LazyMount minHeight={360} rootMargin="400px">
|
||||||
objId={objId}
|
<ObjectInfraMap
|
||||||
centerLat={d?.latitude ?? null}
|
objId={objId}
|
||||||
centerLon={d?.longitude ?? null}
|
centerLat={d?.latitude ?? null}
|
||||||
/>
|
centerLon={d?.longitude ?? null}
|
||||||
|
/>
|
||||||
|
</LazyMount>
|
||||||
</Section>
|
</Section>
|
||||||
|
|
||||||
<Section
|
<Section
|
||||||
title="Фото прогресса строительства"
|
title="Фото прогресса строительства"
|
||||||
subtitle="DOM.РФ /construction/progress/photo. Клик по фото — увеличение."
|
subtitle="DOM.РФ /construction/progress/photo. Клик по фото — увеличение."
|
||||||
>
|
>
|
||||||
<ObjectPhotoGallery objId={objId} />
|
<LazyMount minHeight={320} rootMargin="500px">
|
||||||
|
<ObjectPhotoGallery objId={objId} />
|
||||||
|
</LazyMount>
|
||||||
</Section>
|
</Section>
|
||||||
</>
|
</>
|
||||||
);
|
);
|
||||||
|
|
|
||||||
53
frontend/src/components/analytics/LazyMount.tsx
Normal file
53
frontend/src/components/analytics/LazyMount.tsx
Normal file
|
|
@ -0,0 +1,53 @@
|
||||||
|
"use client";
|
||||||
|
|
||||||
|
import { useEffect, useRef, useState } from "react";
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
children: React.ReactNode;
|
||||||
|
/** Pixel offset for rootMargin so we mount slightly before scroll arrives. */
|
||||||
|
rootMargin?: string;
|
||||||
|
/** Reserved height while not yet mounted, prevents CLS. */
|
||||||
|
minHeight?: number;
|
||||||
|
/** If true, render immediately without waiting for intersection. */
|
||||||
|
eager?: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function LazyMount({
|
||||||
|
children,
|
||||||
|
rootMargin = "300px",
|
||||||
|
minHeight = 320,
|
||||||
|
eager = false,
|
||||||
|
}: Props) {
|
||||||
|
const ref = useRef<HTMLDivElement | null>(null);
|
||||||
|
const [visible, setVisible] = useState(eager);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (visible) return;
|
||||||
|
if (typeof IntersectionObserver === "undefined") {
|
||||||
|
setVisible(true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const el = ref.current;
|
||||||
|
if (!el) return;
|
||||||
|
const io = new IntersectionObserver(
|
||||||
|
(entries) => {
|
||||||
|
for (const e of entries) {
|
||||||
|
if (e.isIntersecting) {
|
||||||
|
setVisible(true);
|
||||||
|
io.disconnect();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{ rootMargin },
|
||||||
|
);
|
||||||
|
io.observe(el);
|
||||||
|
return () => io.disconnect();
|
||||||
|
}, [visible, rootMargin]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div ref={ref} style={{ minHeight: visible ? undefined : minHeight }}>
|
||||||
|
{visible ? children : null}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
@ -86,7 +86,18 @@ export function ObjectInfraMap({ objId, centerLat, centerLon }: Props) {
|
||||||
</select>
|
</select>
|
||||||
</div>
|
</div>
|
||||||
{isLoading ? (
|
{isLoading ? (
|
||||||
<div style={{ padding: 24, color: "#5b6066" }}>Загрузка POI…</div>
|
<div
|
||||||
|
style={{
|
||||||
|
padding: 24,
|
||||||
|
color: "#5b6066",
|
||||||
|
minHeight: 320,
|
||||||
|
background: "#fafbfc",
|
||||||
|
border: "1px solid #eef0f3",
|
||||||
|
borderRadius: 8,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
Загрузка POI…
|
||||||
|
</div>
|
||||||
) : (
|
) : (
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,5 @@
|
||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import Image from "next/image";
|
|
||||||
import { useEffect, useState } from "react";
|
import { useEffect, useState } from "react";
|
||||||
|
|
||||||
import { useObjectPhotos } from "@/lib/analytics-api";
|
import { useObjectPhotos } from "@/lib/analytics-api";
|
||||||
|
|
@ -22,12 +21,36 @@ export function ObjectPhotoGallery({ objId }: { objId: number | string }) {
|
||||||
}, [active]);
|
}, [active]);
|
||||||
|
|
||||||
if (isLoading) {
|
if (isLoading) {
|
||||||
return <div style={{ color: "#5b6066" }}>Загрузка фото…</div>;
|
return (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
display: "grid",
|
||||||
|
gridTemplateColumns: "repeat(auto-fill, minmax(140px, 1fr))",
|
||||||
|
gap: 8,
|
||||||
|
minHeight: 320,
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{Array.from({ length: 12 }).map((_, i) => (
|
||||||
|
<div
|
||||||
|
key={i}
|
||||||
|
style={{
|
||||||
|
aspectRatio: "4/3",
|
||||||
|
background: "#f3f4f6",
|
||||||
|
borderRadius: 6,
|
||||||
|
animation: "pulse 1.4s ease-in-out infinite",
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
<style>{`@keyframes pulse {0%,100%{opacity:1}50%{opacity:.55}}`}</style>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const photos = (data ?? []).filter((p) => p.photo_url);
|
const photos = (data ?? []).filter((p) => p.photo_url);
|
||||||
if (photos.length === 0) {
|
if (photos.length === 0) {
|
||||||
return <div style={{ color: "#9ca3af" }}>Фото отсутствуют.</div>;
|
return (
|
||||||
|
<div style={{ color: "#9ca3af", minHeight: 80 }}>Фото отсутствуют.</div>
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const visible = showAll ? photos : photos.slice(0, INITIAL_LIMIT);
|
const visible = showAll ? photos : photos.slice(0, INITIAL_LIMIT);
|
||||||
|
|
@ -51,7 +74,7 @@ export function ObjectPhotoGallery({ objId }: { objId: number | string }) {
|
||||||
gap: 8,
|
gap: 8,
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
{visible.map((p) => (
|
{visible.map((p, i) => (
|
||||||
<button
|
<button
|
||||||
key={p.obj_file_id}
|
key={p.obj_file_id}
|
||||||
onClick={() => setActive(p.obj_file_id)}
|
onClick={() => setActive(p.obj_file_id)}
|
||||||
|
|
@ -66,15 +89,19 @@ export function ObjectPhotoGallery({ objId }: { objId: number | string }) {
|
||||||
position: "relative",
|
position: "relative",
|
||||||
}}
|
}}
|
||||||
>
|
>
|
||||||
<Image
|
{/* eslint-disable-next-line @next/next/no-img-element */}
|
||||||
src={p.photo_url!}
|
<img
|
||||||
|
src={p.thumb_url}
|
||||||
alt={p.photo_name ?? "фото прогресса"}
|
alt={p.photo_name ?? "фото прогресса"}
|
||||||
fill
|
loading={i < 6 ? "eager" : "lazy"}
|
||||||
sizes="160px"
|
decoding="async"
|
||||||
quality={60}
|
style={{
|
||||||
style={{ objectFit: "cover" }}
|
width: "100%",
|
||||||
loading="lazy"
|
height: "100%",
|
||||||
unoptimized={false}
|
objectFit: "cover",
|
||||||
|
position: "absolute",
|
||||||
|
inset: 0,
|
||||||
|
}}
|
||||||
/>
|
/>
|
||||||
<div
|
<div
|
||||||
style={{
|
style={{
|
||||||
|
|
@ -129,7 +156,7 @@ export function ObjectPhotoGallery({ objId }: { objId: number | string }) {
|
||||||
>
|
>
|
||||||
{/* eslint-disable-next-line @next/next/no-img-element */}
|
{/* eslint-disable-next-line @next/next/no-img-element */}
|
||||||
<img
|
<img
|
||||||
src={activePhoto.photo_url!}
|
src={activePhoto.full_url}
|
||||||
alt={activePhoto.photo_name ?? ""}
|
alt={activePhoto.photo_name ?? ""}
|
||||||
style={{
|
style={{
|
||||||
maxWidth: "92vw",
|
maxWidth: "92vw",
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,6 @@
|
||||||
"use client";
|
"use client";
|
||||||
|
|
||||||
import { useQuery } from "@tanstack/react-query";
|
import { keepPreviousData, useQuery } from "@tanstack/react-query";
|
||||||
|
|
||||||
import { apiFetch } from "./api";
|
import { apiFetch } from "./api";
|
||||||
import type {
|
import type {
|
||||||
|
|
@ -194,6 +194,7 @@ export function useObjectInfrastructure(
|
||||||
);
|
);
|
||||||
},
|
},
|
||||||
enabled: !!objId,
|
enabled: !!objId,
|
||||||
|
placeholderData: keepPreviousData,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -183,6 +183,8 @@ export interface ObjectPhoto {
|
||||||
obj_file_id: string;
|
obj_file_id: string;
|
||||||
ord_num: number | null;
|
ord_num: number | null;
|
||||||
photo_url: string | null;
|
photo_url: string | null;
|
||||||
|
thumb_url: string;
|
||||||
|
full_url: string;
|
||||||
photo_dttm: string | null;
|
photo_dttm: string | null;
|
||||||
period_dt: string | null;
|
period_dt: string | null;
|
||||||
size_bytes: number | null;
|
size_bytes: number | null;
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
Loading…
Add table
Reference in a new issue