From c3bc96535de6b28685be0ac080f8a0d4cb0155ef Mon Sep 17 00:00:00 2001 From: lekss361 Date: Tue, 28 Apr 2026 23:17:42 +0300 Subject: [PATCH] =?UTF-8?q?fix(scraper):=20=D0=B4=D0=B8=D0=B0=D0=B3=D0=BD?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D0=B8=D1=80=D1=83=D0=B5=D0=BC=20=C2=AB=D0=BA?= =?UTF-8?q?=D0=BD=D0=BE=D0=BF=D0=BA=D0=B0=20=D0=BD=D0=B5=20=D1=80=D0=B0?= =?UTF-8?q?=D0=B1=D0=BE=D1=82=D0=B0=D0=B5=D1=82=C2=BB=20=E2=80=94=20Redis?= =?UTF-8?q?=20lock=20+=20task=5Freceived=20log?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Симптом: POST /admin/scrape/kn возвращает task_id, но run_id не появляется в kn_scrape_runs. Причина: зомби-worker оставил Redis-lock с TTL=2h, новые задачи скипаются с reason='lock_held' молча. Backend: - _LOCK_TTL_SECONDS 7200 → 1800 (full sweep ~25-30 мин — потолок). - force_release_lock() helper + новый POST /api/v1/admin/scrape/release-lock. - TriggerKnRequest +force: bool — при true перед apply_async удалить lock. - _log_task_received() / _log_task_skipped() — kn_scrape_log пишет «task подхвачен/skip» с run_id=NULL до основного flow. Теперь UI показывает whether worker реально получил задачу. Frontend: - Чекбокс ⚡ Force в форме запуска. - Кнопка 🔓 Снять Redis-lock (отдельный endpoint). - Уведомление о результате release. --- .gitignore | 3 + backend/.dockerignore | 29 ++++++++ backend/Dockerfile | 23 +++--- backend/app/api/v1/admin_scrape.py | 37 +++++++++- backend/app/workers/tasks/scrape_kn.py | 99 ++++++++++++++++++++++++-- frontend/.dockerignore | 29 ++++++++ frontend/src/app/admin/scrape/page.tsx | 96 ++++++++++++++++++++++++- frontend/tsconfig.tsbuildinfo | 1 - 8 files changed, 298 insertions(+), 19 deletions(-) delete mode 100644 frontend/tsconfig.tsbuildinfo diff --git a/.gitignore b/.gitignore index 6953ac2e..14ae026d 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,9 @@ node_modules/ .next/ frontend/.next/ out/ +# TypeScript incremental build info — must NOT be committed; locally-generated +# and changes on every build, which busts Docker build context cache. +*.tsbuildinfo # Env / secrets .env diff --git a/backend/.dockerignore b/backend/.dockerignore index a7657b4a..35b4d8e3 100644 --- a/backend/.dockerignore +++ b/backend/.dockerignore @@ -1,3 +1,7 @@ +# backend Dockerfile делает целевой `COPY app ./app` (не COPY .), поэтому +# контекст и так чистый. Но build context передаётся ЦЕЛИКОМ от docker buildx +# в daemon/builder — лишние мегабайты замедляют upload и хешинг. + .venv __pycache__ *.pyc @@ -8,3 +12,28 @@ tests .env .env.* !.env.example + +# VCS +.git +.gitignore +.gitattributes + +# Local data (raw downloads — никогда не нужны в образе) +data/raw +data/raw_endpoints +data/photos + +# Logs / coverage +*.log +.coverage +htmlcov + +# Docs / IDE +*.md +docs +.vscode +.idea + +# OS +.DS_Store +Thumbs.db diff --git a/backend/Dockerfile b/backend/Dockerfile index 6c909527..8e15efd4 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -15,20 +15,23 @@ ENV PYTHONUNBUFFERED=1 \ UV_COMPILE_BYTECODE=1 \ UV_LINK_MODE=copy -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y --no-install-recommends \ build-essential \ libpq-dev \ libgdal-dev \ libproj-dev \ - libgeos-dev \ - && rm -rf /var/lib/apt/lists/* + libgeos-dev RUN pip install --no-cache-dir uv WORKDIR /app COPY pyproject.toml uv.lock* ./ -RUN if [ -f uv.lock ]; then uv sync --frozen --no-dev; else uv sync --no-dev; fi +# uv-кеш переживает между билдами: при неизменном lock'е sync ~5 сек вместо 1-2 мин. +RUN --mount=type=cache,target=/root/.cache/uv \ + if [ -f uv.lock ]; then uv sync --frozen --no-dev; else uv sync --no-dev; fi COPY app ./app COPY alembic.ini ./ @@ -42,7 +45,9 @@ ENV PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ PATH="/app/.venv/bin:$PATH" -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y --no-install-recommends \ libpq5 \ libgdal36 \ libproj25 \ @@ -52,7 +57,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libpangoft2-1.0-0 \ fonts-dejavu-core \ curl \ - && rm -rf /var/lib/apt/lists/* \ && useradd --create-home --uid 1000 app WORKDIR /app @@ -75,7 +79,9 @@ FROM runner AS runner-with-chromium USER root -RUN apt-get update && apt-get install -y --no-install-recommends \ +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y --no-install-recommends \ # Chromium runtime libs (раньше заходили через `playwright install --with-deps`) libnss3 \ libnspr4 \ @@ -97,7 +103,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ENV PLAYWRIGHT_BROWSERS_PATH=/opt/playwright RUN mkdir -p $PLAYWRIGHT_BROWSERS_PATH \ && /app/.venv/bin/playwright install chromium \ - && chmod -R a+rX $PLAYWRIGHT_BROWSERS_PATH \ - && rm -rf /var/lib/apt/lists/* + && chmod -R a+rX $PLAYWRIGHT_BROWSERS_PATH USER app diff --git a/backend/app/api/v1/admin_scrape.py b/backend/app/api/v1/admin_scrape.py index f3f1f473..1edd2bd9 100644 --- a/backend/app/api/v1/admin_scrape.py +++ b/backend/app/api/v1/admin_scrape.py @@ -29,6 +29,9 @@ class TriggerKnRequest(BaseModel): fetch_flats: bool = True extras: bool = True download_photos: bool = False + # Если True — ПЕРЕД apply_async удалить existing Redis lock. Эмердженси-режим + # для случая «зомби-worker оставил lock и блокирует новые задачи». + force: bool = False def _check_token(x_admin_token: str | None) -> None: @@ -48,7 +51,14 @@ def trigger_kn_sweep( ) -> dict[str, Any]: _check_token(x_admin_token) # Defer Celery import so the API works without a broker in dev. - from app.workers.tasks.scrape_kn import scrape_kn_region + from app.workers.tasks.scrape_kn import ( + force_release_lock, + scrape_kn_region, + ) + + lock_released = False + if payload.force: + lock_released = force_release_lock(payload.region_code, payload.developers) result = scrape_kn_region.apply_async( args=[payload.region_code, payload.developers], @@ -64,6 +74,8 @@ def trigger_kn_sweep( "region_code": payload.region_code, "developers": payload.developers, "queued_at": result.date_done.isoformat() if result.date_done else None, + "force": payload.force, + "lock_was_released": lock_released, } @@ -208,6 +220,29 @@ def queue_status( } +class ReleaseLockRequest(BaseModel): + region_code: int = Field(..., ge=1, le=99) + developers: list[str] | None = Field(default=None, max_length=20) + + +@router.post("/release-lock") +def release_lock( + payload: ReleaseLockRequest, + x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None, +) -> dict[str, Any]: + """Принудительно удалить Redis-lock для (region, devs). Использовать когда + зомби-worker оставил lock и новые задачи скипаются с reason='lock_held'.""" + _check_token(x_admin_token) + from app.workers.tasks.scrape_kn import force_release_lock + + released = force_release_lock(payload.region_code, payload.developers) + return { + "region_code": payload.region_code, + "developers": payload.developers, + "released": released, + } + + class RevokeRequest(BaseModel): task_id: str = Field(..., min_length=1, max_length=255) terminate: bool = False # SIGTERM running task vs just unmark from queue diff --git a/backend/app/workers/tasks/scrape_kn.py b/backend/app/workers/tasks/scrape_kn.py index c3e75f37..0e37afa8 100644 --- a/backend/app/workers/tasks/scrape_kn.py +++ b/backend/app/workers/tasks/scrape_kn.py @@ -18,18 +18,94 @@ from app.workers.celery_app import celery_app logger = logging.getLogger(__name__) -# Lock TTL longer than the longest expected sweep (442-obj sweep with extras -# can run ~30-60 min). 2 hours is a safe upper bound — after that we assume -# the worker died and let a new run proceed. -_LOCK_TTL_SECONDS = 2 * 60 * 60 +# Lock TTL — full Свердл sweep с extras занимает ~25-30 мин. 30 мин — потолок. +# Если worker умер с lock'ом — он истечёт за полчаса, а не за 2 часа (раньше). +# Heartbeat-based autoclean в /queue endpoint всё равно подхватит зомби-runs. +_LOCK_TTL_SECONDS = 30 * 60 + + +def _lock_key(region_code: int, developers: list[str] | None) -> str: + devs_key = ",".join(developers) if developers else "*" + return f"scrape:kn:lock:{region_code}:{devs_key}" + + +def _log_task_received(region_code: int, devs: list[str] | None, task_id: str | None) -> None: + """Записать в kn_scrape_log что worker получил задачу. Без run_id — + он появится позже в run_region_sweep. Best-effort: SQL ошибки гасим.""" + from sqlalchemy import text + + from app.core.db import SessionLocal + + db = SessionLocal() + try: + db.execute( + text( + """ + INSERT INTO kn_scrape_log (run_id, level, stage, message) + VALUES (NULL, 'info', 'task_received', :msg) + """ + ), + { + "msg": ( + f"Worker подхватил task_id={task_id or '?'}, region={region_code}," + f" devs={devs or '*'}" + )[:500], + }, + ) + db.commit() + except Exception as e: + logger.warning("log_task_received failed: %s", e) + finally: + db.close() + + +def _log_task_skipped( + region_code: int, devs: list[str] | None, task_id: str | None, reason: str +) -> None: + from sqlalchemy import text + + from app.core.db import SessionLocal + + db = SessionLocal() + try: + db.execute( + text( + """ + INSERT INTO kn_scrape_log (run_id, level, stage, message) + VALUES (NULL, 'warn', 'task_skipped', :msg) + """ + ), + { + "msg": ( + f"task_id={task_id or '?'} skipped: {reason}, region={region_code}," + f" devs={devs or '*'}" + )[:500], + }, + ) + db.commit() + except Exception as e: + logger.warning("log_task_skipped failed: %s", e) + finally: + db.close() + + +def force_release_lock(region_code: int, developers: list[str] | None) -> bool: + """Принудительно удалить Redis-lock (для emergency manual trigger, + когда зомби-worker оставил key). Возвращает True если ключ был.""" + key = _lock_key(region_code, developers) + r = redis.Redis.from_url(settings.redis_url) + try: + return bool(r.delete(key)) + except Exception as e: + logger.warning("force_release_lock %s failed: %s", key, e) + return False @contextmanager def _region_lock(region_code: int, developers: list[str] | None): """Redis singleton lock so beat double-fire or two workers can't run the same (region, devs) sweep concurrently. SET NX EX is atomic.""" - devs_key = ",".join(developers) if developers else "*" - key = f"scrape:kn:lock:{region_code}:{devs_key}" + key = _lock_key(region_code, developers) r = redis.Redis.from_url(settings.redis_url) acquired = r.set(key, "1", nx=True, ex=_LOCK_TTL_SECONDS) try: @@ -59,6 +135,12 @@ def scrape_kn_region( DOM.РФ at exactly the same minute. Manual triggers pass skip_jitter=True for immediate execution. """ + # Сразу пишем «task received» в kn_scrape_log БЕЗ run_id (run_id ещё не + # создан — он появится в run_region_sweep). Это позволяет UI видеть + # «worker подхватил задачу» даже если потом сработает lock_held или + # упадём ещё до Phase A. + _log_task_received(region_code, developers, self.request.id) + if not skip_jitter and settings.scrape_kn_jitter_seconds > 0: delay = random.uniform(0, settings.scrape_kn_jitter_seconds) logger.info("jitter delay %.0fs before scrape", delay) @@ -77,11 +159,13 @@ def scrape_kn_region( region_code, developers, ) + _log_task_skipped(region_code, developers, self.request.id, "lock_held") return { "skipped": True, "reason": "lock_held", "region_code": region_code, "developers": developers, + "lock_key": _lock_key(region_code, developers), } logger.info( @@ -145,6 +229,8 @@ def resume_kn_run(self: Any, run_id: int) -> dict[str, Any]: return {"skipped": True, "reason": "no_region", "run_id": run_id} region_code = region_codes[0] + _log_task_received(region_code, developers, self.request.id) + with _region_lock(region_code, developers) as got_lock: if not got_lock: logger.warning( @@ -152,6 +238,7 @@ def resume_kn_run(self: Any, run_id: int) -> dict[str, Any]: run_id, region_code, ) + _log_task_skipped(region_code, developers, self.request.id, "lock_held") return {"skipped": True, "reason": "lock_held", "run_id": run_id} logger.info("resume_kn_run start run=%s region=%s devs=%s", run_id, region_code, developers) diff --git a/frontend/.dockerignore b/frontend/.dockerignore index 6e656fb0..3399cfff 100644 --- a/frontend/.dockerignore +++ b/frontend/.dockerignore @@ -1,6 +1,35 @@ +# Каждое лишнее изменение в build context инвалидирует layer cache в Docker. +# Здесь — только то, что НЕ нужно для `npm ci` + `next build`. + node_modules .next +.next/cache .env .env.* !.env.example .eslintcache +*.tsbuildinfo + +# VCS +.git +.gitignore +.gitattributes + +# Logs / coverage / IDE +*.log +coverage +.vscode +.idea + +# Tests + docs (не нужны для production build) +__tests__ +**/*.test.ts +**/*.test.tsx +**/*.spec.ts +**/*.spec.tsx +*.md +docs + +# OS +.DS_Store +Thumbs.db diff --git a/frontend/src/app/admin/scrape/page.tsx b/frontend/src/app/admin/scrape/page.tsx index aa761421..48189ee2 100644 --- a/frontend/src/app/admin/scrape/page.tsx +++ b/frontend/src/app/admin/scrape/page.tsx @@ -76,6 +76,7 @@ export default function ScrapeAdminPage() { const [fetchFlats, setFetchFlats] = useState(true); const [extras, setExtras] = useState(true); const [downloadPhotos, setDownloadPhotos] = useState(false); + const [forceRun, setForceRun] = useState(false); const [showFailures, setShowFailures] = useState(false); const [failuresRunId, setFailuresRunId] = useState(""); const [logsRunId, setLogsRunId] = useState(""); @@ -107,6 +108,7 @@ export default function ScrapeAdminPage() { fetch_flats: fetchFlats, extras, download_photos: downloadPhotos, + force: forceRun, }), }), onSuccess: (data) => { @@ -118,6 +120,35 @@ export default function ScrapeAdminPage() { }, }); + const releaseLock = useMutation({ + mutationFn: () => + apiFetch<{ + region_code: number; + developers: string[] | null; + released: boolean; + }>("/api/v1/admin/scrape/release-lock", { + method: "POST", + headers: { "X-Admin-Token": token }, + body: JSON.stringify({ + region_code: region, + developers: devs + ? devs + .split(",") + .map((s) => s.trim()) + .filter(Boolean) + : null, + }), + }), + onSuccess: (data) => { + console.log("[scrape] lock released:", data); + queryClient.invalidateQueries({ queryKey: ["scrape-queue"] }); + }, + onError: (err) => { + console.error("[scrape] release-lock failed:", err); + alert(`Не удалось снять lock: ${String(err)}`); + }, + }); + const queue = useQuery({ queryKey: ["scrape-queue", token], queryFn: () => @@ -297,8 +328,28 @@ export default function ScrapeAdminPage() { /> Скачать фото (binary) + -
+
+ {!token ? ( - + ← введите Admin Token выше ) : null}
+ {releaseLock.isSuccess ? ( +
+ {releaseLock.data.released + ? `✓ Lock снят для region=${releaseLock.data.region_code} devs=${releaseLock.data.developers ?? "*"}. Можно запускать.` + : `Lock не был активен (region=${releaseLock.data.region_code}) — путь чист.`} +
+ ) : null} {trigger.isError ? (