From b3c15bb8ea055d329f5a1d48c7e3c7d5496a31d2 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Mon, 27 Apr 2026 20:26:55 +0300 Subject: [PATCH] fix ui lag --- backend/app/api/v1/admin_scrape.py | 101 +++++++++++++----- backend/app/api/v1/photos.py | 87 +++++++++++++++ backend/app/main.py | 3 +- backend/app/services/analytics_queries.py | 4 + backend/app/services/photos/__init__.py | 0 backend/app/services/photos/thumbs.py | 57 ++++++++++ backend/app/services/scrapers/domrf_kn.py | 61 +++++++---- backend/pyproject.toml | 1 + data/sql/53_schema_kn_thumbs.sql | 10 ++ frontend/src/app/admin/scrape/page.tsx | 18 ++++ .../src/app/analytics/objects/[id]/page.tsx | 77 +++++++++++-- .../src/components/analytics/LazyMount.tsx | 53 +++++++++ .../components/analytics/ObjectInfraMap.tsx | 13 ++- .../analytics/ObjectPhotoGallery.tsx | 53 ++++++--- frontend/src/lib/analytics-api.ts | 3 +- frontend/src/types/analytics.ts | 2 + frontend/tsconfig.tsbuildinfo | 2 +- 17 files changed, 473 insertions(+), 72 deletions(-) create mode 100644 backend/app/api/v1/photos.py create mode 100644 backend/app/services/photos/__init__.py create mode 100644 backend/app/services/photos/thumbs.py create mode 100644 data/sql/53_schema_kn_thumbs.sql create mode 100644 frontend/src/components/analytics/LazyMount.tsx diff --git a/backend/app/api/v1/admin_scrape.py b/backend/app/api/v1/admin_scrape.py index 3badc6e4..30e2b6ae 100644 --- a/backend/app/api/v1/admin_scrape.py +++ b/backend/app/api/v1/admin_scrape.py @@ -69,17 +69,65 @@ def trigger_kn_sweep( @router.get("/queue") def queue_status( + db: Annotated[Session, Depends(get_db)], x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None, ) -> dict[str, Any]: - """Snapshot of Celery worker(s) state: active tasks, reserved (queued on - worker, not yet started), scheduled (with ETA), beat-schedule, and queue - depth in Redis. If no worker is running, all worker-side fields are empty. + """Snapshot of Celery state, optimised for UI poll latency. + + - active: from kn_scrape_runs WHERE status='running' (instant, richer info + than celery inspect — includes objects_count/flats_count/requests_count). + - reserved + workers: celery inspect with short timeout (500 ms), called in + a thread so a missing worker can't block the request. + - queue_depth: Redis LLEN on default queue (single round-trip). + - scheduled: omitted — we don't use ETA tasks. + + Worst-case latency ≈ 600 ms even if no worker is reachable, vs ~8 s with + the previous 4×2 s blocking inspect calls. """ _check_token(x_admin_token) + import concurrent.futures + import time + from app.workers.celery_app import celery_app - inspect = celery_app.control.inspect(timeout=2.0) + # 1) Active runs straight from DB — no broker round-trip. + active_rows = ( + db.execute( + text( + """ + SELECT run_id, started_at, region_codes, developer_ids, + objects_count, flats_count, requests_count + FROM kn_scrape_runs + WHERE status = 'running' + ORDER BY started_at DESC + LIMIT 20 + """ + ) + ) + .mappings() + .all() + ) + active: list[dict[str, Any]] = [ + { + "worker": "db", + "task_id": f"run-{r['run_id']}", + "name": "scrape_kn_region", + "args": [list(r["region_codes"] or []), list(r["developer_ids"] or [])], + "kwargs": { + "objects": r["objects_count"], + "flats": r["flats_count"], + "requests": r["requests_count"], + }, + "time_start": (int(r["started_at"].timestamp()) if r["started_at"] else None), + "eta": None, + } + for r in active_rows + ] + + # 2) Run reserved + ping in parallel with a tight timeout so the endpoint + # never hangs when the broker is unreachable or worker is gone. + inspect = celery_app.control.inspect(timeout=0.5) def _flatten(per_worker: dict[str, list[dict[str, Any]]] | None) -> list[dict[str, Any]]: if not per_worker: @@ -100,42 +148,43 @@ def queue_status( ) return out - try: - active = _flatten(inspect.active()) - except Exception: - active = [] - try: - reserved = _flatten(inspect.reserved()) - except Exception: - reserved = [] - try: - scheduled = _flatten(inspect.scheduled()) - except Exception: - scheduled = [] + def _safe(fn): + try: + return fn() + except Exception: + return None - # Pending in broker queue (not yet picked up by any worker). + deadline = time.monotonic() + 0.8 + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as ex: + f_reserved = ex.submit(_safe, inspect.reserved) + f_ping = ex.submit(_safe, inspect.ping) + try: + reserved_raw = f_reserved.result(timeout=max(0.1, deadline - time.monotonic())) + except concurrent.futures.TimeoutError: + reserved_raw = None + try: + ping_resp = f_ping.result(timeout=max(0.1, deadline - time.monotonic())) + except concurrent.futures.TimeoutError: + ping_resp = None + + reserved = _flatten(reserved_raw) + workers = list((ping_resp or {}).keys()) + + # 3) Pending in broker queue (not yet picked up by any worker). queue_depth: int | None = None try: with celery_app.connection_or_acquire() as conn: with conn.channel() as channel: - # Default queue name is 'celery'. queue_depth = channel.client.llen("celery") except Exception: queue_depth = None - workers: list[str] = [] - try: - ping_resp = inspect.ping() or {} - workers = list(ping_resp.keys()) - except Exception: - pass - return { "workers": workers, "queue_depth": queue_depth, "active": active, "reserved": reserved, - "scheduled": scheduled, + "scheduled": [], } diff --git a/backend/app/api/v1/photos.py b/backend/app/api/v1/photos.py new file mode 100644 index 00000000..ab06f637 --- /dev/null +++ b/backend/app/api/v1/photos.py @@ -0,0 +1,87 @@ +"""Serve cached DOM.РФ photos. + +GET /api/v1/photos/{obj_id}/{file_id}?size=thumb|full + +- size=thumb (default): WebP 320×240 from local cache. If thumb is missing + but the original is present locally, generate one on the fly. If neither + is present, redirect to the upstream DOM.РФ URL. +- size=full: original PNG/JPG if cached locally, else redirect upstream. + +This isolates the frontend grid from upstream latency and from Next.js dev-mode +optimizer cold-hit cost. +""" + +from __future__ import annotations + +import logging +from pathlib import Path +from typing import Annotated, Literal + +from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi.responses import FileResponse, RedirectResponse, Response +from sqlalchemy import text +from sqlalchemy.orm import Session + +from app.core.db import get_db +from app.services.photos.thumbs import make_thumbnail + +logger = logging.getLogger(__name__) +router = APIRouter() + + +_PHOTO_LOOKUP_SQL = text( + """ + SELECT local_path, thumb_path, photo_url + FROM domrf_kn_photos + WHERE obj_id = :obj AND obj_file_id = :fid + LIMIT 1 + """ +) + + +@router.get("/{obj_id}/{file_id}") +def get_photo( + db: Annotated[Session, Depends(get_db)], + obj_id: int, + file_id: str, + size: Annotated[Literal["thumb", "full"], Query()] = "thumb", +) -> Response: + row = db.execute(_PHOTO_LOOKUP_SQL, {"obj": obj_id, "fid": file_id}).mappings().first() + if not row: + raise HTTPException(status_code=404, detail="photo not registered") + + local_path = row["local_path"] + thumb_path = row["thumb_path"] + upstream = row["photo_url"] + + headers = {"Cache-Control": "public, max-age=604800, immutable"} + + if size == "full": + if local_path and Path(local_path).exists(): + return FileResponse(local_path, headers=headers) + if upstream: + return RedirectResponse(upstream, status_code=302) + raise HTTPException(status_code=404, detail="full photo unavailable") + + # size=thumb + if thumb_path and Path(thumb_path).exists(): + return FileResponse(thumb_path, media_type="image/webp", headers=headers) + + # Generate thumb on the fly from cached original + if local_path and Path(local_path).exists(): + generated = make_thumbnail(Path(local_path)) + if generated and generated.exists(): + db.execute( + text( + "UPDATE domrf_kn_photos SET thumb_path = :tp" + " WHERE obj_id = :o AND obj_file_id = :f" + ), + {"tp": str(generated), "o": obj_id, "f": file_id}, + ) + db.commit() + return FileResponse(str(generated), media_type="image/webp", headers=headers) + + # Last resort — redirect to upstream so the page still renders + if upstream: + return RedirectResponse(upstream, status_code=302) + raise HTTPException(status_code=404, detail="photo unavailable") diff --git a/backend/app/main.py b/backend/app/main.py index 8203b642..768d31ac 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -5,7 +5,7 @@ import sentry_sdk from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from app.api.v1 import admin_leads, admin_scrape, analytics, concepts, parcels +from app.api.v1 import admin_leads, admin_scrape, analytics, concepts, parcels, photos from app.core.config import settings @@ -35,6 +35,7 @@ app.include_router(parcels.router, prefix="/api/v1/parcels", tags=["parcels"]) app.include_router(analytics.router, prefix="/api/v1/analytics", tags=["analytics"]) app.include_router(admin_scrape.router, prefix="/api/v1/admin/scrape", tags=["admin"]) app.include_router(admin_leads.router, prefix="/api/v1/admin/leads", tags=["admin"]) +app.include_router(photos.router, prefix="/api/v1/photos", tags=["photos"]) @app.get("/health") diff --git a/backend/app/services/analytics_queries.py b/backend/app/services/analytics_queries.py index 1dddf81e..2ceaabb2 100644 --- a/backend/app/services/analytics_queries.py +++ b/backend/app/services/analytics_queries.py @@ -787,6 +787,10 @@ def object_photos(db: Session, obj_id: int, limit: int = 100) -> list[dict[str, "obj_file_id": r["obj_file_id"], "ord_num": r["ord_num"], "photo_url": r["photo_url"], + # Always serve thumbs through our backend — cached WebP, no upstream + # latency, no Next.js dev-mode optimizer cold-hit cost. + "thumb_url": f"/api/v1/photos/{obj_id}/{r['obj_file_id']}?size=thumb", + "full_url": f"/api/v1/photos/{obj_id}/{r['obj_file_id']}?size=full", "photo_dttm": r["photo_dttm"].isoformat() if r["photo_dttm"] else None, "period_dt": r["period_dt"].isoformat() if r["period_dt"] else None, "size_bytes": r["size_bytes"], diff --git a/backend/app/services/photos/__init__.py b/backend/app/services/photos/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/app/services/photos/thumbs.py b/backend/app/services/photos/thumbs.py new file mode 100644 index 00000000..5b79e22e --- /dev/null +++ b/backend/app/services/photos/thumbs.py @@ -0,0 +1,57 @@ +"""Thumbnail generation for DOM.РФ photos. + +Strategy: original PNG/JPG ~1-3 MB downloaded by scraper to +data/raw/domrf_photos//.{png,jpg}. We additionally write a +WebP cover-thumbnail at 320×240 (~10-30 KB) next to it, suffix `_thumb.webp`. + +The frontend grid uses /api/v1/photos//?size=thumb. Lightbox +opens the original DOM.РФ URL (we don't need to mirror originals). +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +from PIL import Image, ImageOps + +logger = logging.getLogger(__name__) + +THUMB_SIZE: tuple[int, int] = (320, 240) +THUMB_QUALITY: int = 70 +THUMB_SUFFIX: str = "_thumb.webp" + + +def thumb_path_for(src: Path) -> Path: + """Return canonical thumbnail path next to the source file.""" + return src.with_name(src.stem + THUMB_SUFFIX) + + +def make_thumbnail( + src: Path, + *, + size: tuple[int, int] = THUMB_SIZE, + quality: int = THUMB_QUALITY, + overwrite: bool = False, +) -> Path | None: + """Generate a WebP cover-thumbnail next to src. Returns thumb path on success. + + Skips if thumb already exists and overwrite=False. Returns None on any error + (logged) so callers can keep going with the next file. + """ + if not src.exists(): + return None + dst = thumb_path_for(src) + if dst.exists() and not overwrite: + return dst + try: + with Image.open(src) as im: + im = ImageOps.exif_transpose(im) + im = im.convert("RGB") + im = ImageOps.fit(im, size, method=Image.Resampling.LANCZOS) + dst.parent.mkdir(parents=True, exist_ok=True) + im.save(dst, format="WEBP", quality=quality, method=4) + return dst + except Exception as e: + logger.warning("thumbnail %s failed: %s", src, e) + return None diff --git a/backend/app/services/scrapers/domrf_kn.py b/backend/app/services/scrapers/domrf_kn.py index 9db5c09a..783f2b4f 100644 --- a/backend/app/services/scrapers/domrf_kn.py +++ b/backend/app/services/scrapers/domrf_kn.py @@ -429,11 +429,11 @@ UPSERT_PHOTO_SQL = text( INSERT INTO domrf_kn_photos ( obj_id, obj_file_id, ord_num, photo_url, photo_dttm, period_dt, size_bytes, photo_name, ready_desc, build_type, hidden, - local_path, downloaded_at + local_path, thumb_path, downloaded_at ) VALUES ( :obj_id, :obj_file_id, :ord_num, :photo_url, :photo_dttm, :period_dt, :size_bytes, :photo_name, :ready_desc, :build_type, :hidden, - :local_path, :downloaded_at + :local_path, :thumb_path, :downloaded_at ) ON CONFLICT (obj_id, obj_file_id) DO UPDATE SET ord_num = EXCLUDED.ord_num, @@ -445,6 +445,7 @@ UPSERT_PHOTO_SQL = text( ready_desc = EXCLUDED.ready_desc, hidden = EXCLUDED.hidden, local_path = COALESCE(EXCLUDED.local_path, domrf_kn_photos.local_path), + thumb_path = COALESCE(EXCLUDED.thumb_path, domrf_kn_photos.thumb_path), downloaded_at = COALESCE(EXCLUDED.downloaded_at, domrf_kn_photos.downloaded_at) """ ) @@ -699,9 +700,11 @@ def upsert_photos( obj_id: int, photos: list[dict[str, Any]], local_paths: dict[str, str] | None = None, + thumb_paths: dict[str, str] | None = None, ) -> int: - """Upsert photo metadata. local_paths maps file_id → relative path on disk.""" + """Upsert photo metadata. local_paths/thumb_paths map file_id → path.""" local_paths = local_paths or {} + thumb_paths = thumb_paths or {} now_dt = datetime.now() n = 0 for p in photos: @@ -709,6 +712,7 @@ def upsert_photos( if not file_id: continue local = local_paths.get(file_id) + thumb = thumb_paths.get(file_id) try: db.execute( UPSERT_PHOTO_SQL, @@ -725,6 +729,7 @@ def upsert_photos( "build_type": p.get("objBuildTypeShortDesc"), "hidden": _to_bool(p.get("objPhotoHiddenFlg")), "local_path": local, + "thumb_path": thumb, "downloaded_at": now_dt if local else None, }, ) @@ -740,12 +745,18 @@ async def download_photos( obj_id: int, photos: list[dict[str, Any]], base_dir: Path, -) -> dict[str, str]: - """Download photo binaries to base_dir/{obj_id}/{file_id}.png (or .jpg). - Returns mapping {file_id: relative_path}. Skips if file already exists with - matching size_bytes. +) -> tuple[dict[str, str], dict[str, str]]: + """Download photo binaries to base_dir/{obj_id}/{file_id}.png (or .jpg) + + generate WebP thumbnails next to them. + + Returns a pair of mappings: (local_paths, thumb_paths) — both keyed by + file_id, value = relative path on disk. Skips originals already present + with matching size_bytes; thumbs skipped if already exist. """ - out: dict[str, str] = {} + from app.services.photos.thumbs import make_thumbnail, thumb_path_for + + locals_out: dict[str, str] = {} + thumbs_out: dict[str, str] = {} obj_dir = base_dir / str(obj_id) obj_dir.mkdir(parents=True, exist_ok=True) for p in photos: @@ -760,15 +771,24 @@ async def download_photos( # Skip if size matches expected_size = p.get("objSize") if local.exists() and expected_size and local.stat().st_size == expected_size: - out[file_id] = str(local) - continue - try: - data = await sess.download_binary(url) - local.write_bytes(data) - out[file_id] = str(local) - except Exception as e: - logger.warning("download photo obj=%s file=%s failed: %s", obj_id, file_id, e) - return out + locals_out[file_id] = str(local) + else: + try: + data = await sess.download_binary(url) + local.write_bytes(data) + locals_out[file_id] = str(local) + except Exception as e: + logger.warning("download photo obj=%s file=%s failed: %s", obj_id, file_id, e) + continue + # Always (re)check thumbnail. make_thumbnail is no-op if it already exists. + existing_thumb = thumb_path_for(local) + if existing_thumb.exists(): + thumbs_out[file_id] = str(existing_thumb) + else: + t = make_thumbnail(local) + if t: + thumbs_out[file_id] = str(t) + return locals_out, thumbs_out # ── orchestrator ───────────────────────────────────────────────────────────── @@ -999,11 +1019,14 @@ async def run_region_sweep( try: photos, full_url = await fetch_photos(sess, obj_id) local_paths: dict[str, str] = {} + thumb_paths: dict[str, str] = {} if download_photos_binary and photos: - local_paths = await download_photos(sess, obj_id, photos, pdir) + local_paths, thumb_paths = await download_photos( + sess, obj_id, photos, pdir + ) extras_counts["photos_downloaded"] += len(local_paths) extras_counts["photos_rows"] += upsert_photos( - db, obj_id, photos, local_paths + db, obj_id, photos, local_paths, thumb_paths ) except Exception as e: full_url = f"{BASE_URL}{PATH_PHOTOS.format(obj_id=obj_id)}" diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 87354014..3e00d611 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -20,6 +20,7 @@ dependencies = [ "celery>=5.4.0", "playwright>=1.45.0", "tenacity>=9.0.0", + "pillow>=10.4.0", "weasyprint>=62.0", "ezdxf>=1.3.0", "openpyxl>=3.1.0", diff --git a/data/sql/53_schema_kn_thumbs.sql b/data/sql/53_schema_kn_thumbs.sql new file mode 100644 index 00000000..6d96f066 --- /dev/null +++ b/data/sql/53_schema_kn_thumbs.sql @@ -0,0 +1,10 @@ +-- Add thumbnail path column for cached locally-served thumbnails. +-- Generated by Pillow at download time (320×240 cover, WebP q=70). +-- Idempotent. + +ALTER TABLE domrf_kn_photos + ADD COLUMN IF NOT EXISTS thumb_path TEXT; + +CREATE INDEX IF NOT EXISTS idx_kn_photos_obj_visible + ON domrf_kn_photos (obj_id) + WHERE COALESCE(hidden, FALSE) = FALSE; diff --git a/frontend/src/app/admin/scrape/page.tsx b/frontend/src/app/admin/scrape/page.tsx index 045c537c..89e38fe9 100644 --- a/frontend/src/app/admin/scrape/page.tsx +++ b/frontend/src/app/admin/scrape/page.tsx @@ -388,6 +388,24 @@ export default function ScrapeAdminPage() {

) : queue.data ? ( <> + {queue.data.active.length === 0 && + queue.data.reserved.length === 0 && + queue.data.scheduled.length === 0 ? ( +
+ ✅ Worker онлайн ({queue.data.workers.join(", ")}) и ждёт задач. + Активных нет, очередь пуста. Нажми «Запустить сейчас» выше или + подожди расписание (cron SCRAPE_KN_CRON). +
+ ) : null} {queue.data.active.length > 0 ? ( <>

+ import("@/components/analytics/ObjectSaleChart").then( + (m) => m.ObjectSaleChart, + ), + { + ssr: false, + loading: () => ( +
+ ), + }, +); + +const ObjectInfraMap = dynamic( + () => + import("@/components/analytics/ObjectInfraMap").then( + (m) => m.ObjectInfraMap, + ), + { + ssr: false, + loading: () => ( +
+ ), + }, +); + +const ObjectPhotoGallery = dynamic( + () => + import("@/components/analytics/ObjectPhotoGallery").then( + (m) => m.ObjectPhotoGallery, + ), + { + ssr: false, + loading: () =>
, + }, +); + export default function ObjectDrillInPage() { const params = useParams<{ id: string }>(); const objId = params.id; @@ -120,25 +171,31 @@ export default function ObjectDrillInPage() { title="Динамика продаж" subtitle="DOM.РФ /sale_graph: реализовано (бары) + средняя цена ₽/м² (линия). Apartments + parking." > - + + +
- + + +
- + + +
); diff --git a/frontend/src/components/analytics/LazyMount.tsx b/frontend/src/components/analytics/LazyMount.tsx new file mode 100644 index 00000000..5866a3d1 --- /dev/null +++ b/frontend/src/components/analytics/LazyMount.tsx @@ -0,0 +1,53 @@ +"use client"; + +import { useEffect, useRef, useState } from "react"; + +interface Props { + children: React.ReactNode; + /** Pixel offset for rootMargin so we mount slightly before scroll arrives. */ + rootMargin?: string; + /** Reserved height while not yet mounted, prevents CLS. */ + minHeight?: number; + /** If true, render immediately without waiting for intersection. */ + eager?: boolean; +} + +export function LazyMount({ + children, + rootMargin = "300px", + minHeight = 320, + eager = false, +}: Props) { + const ref = useRef(null); + const [visible, setVisible] = useState(eager); + + useEffect(() => { + if (visible) return; + if (typeof IntersectionObserver === "undefined") { + setVisible(true); + return; + } + const el = ref.current; + if (!el) return; + const io = new IntersectionObserver( + (entries) => { + for (const e of entries) { + if (e.isIntersecting) { + setVisible(true); + io.disconnect(); + break; + } + } + }, + { rootMargin }, + ); + io.observe(el); + return () => io.disconnect(); + }, [visible, rootMargin]); + + return ( +
+ {visible ? children : null} +
+ ); +} diff --git a/frontend/src/components/analytics/ObjectInfraMap.tsx b/frontend/src/components/analytics/ObjectInfraMap.tsx index 7beca070..f4bd1c0d 100644 --- a/frontend/src/components/analytics/ObjectInfraMap.tsx +++ b/frontend/src/components/analytics/ObjectInfraMap.tsx @@ -86,7 +86,18 @@ export function ObjectInfraMap({ objId, centerLat, centerLon }: Props) {
{isLoading ? ( -
Загрузка POI…
+
+ Загрузка POI… +
) : (
Загрузка фото…
; + return ( +
+ {Array.from({ length: 12 }).map((_, i) => ( +
+ ))} + +
+ ); } const photos = (data ?? []).filter((p) => p.photo_url); if (photos.length === 0) { - return
Фото отсутствуют.
; + return ( +
Фото отсутствуют.
+ ); } const visible = showAll ? photos : photos.slice(0, INITIAL_LIMIT); @@ -51,7 +74,7 @@ export function ObjectPhotoGallery({ objId }: { objId: number | string }) { gap: 8, }} > - {visible.map((p) => ( + {visible.map((p, i) => (