fix(scraper): диагностируем «кнопка не работает» — Redis lock + task_received log

Симптом: POST /admin/scrape/kn возвращает task_id, но run_id не появляется
в kn_scrape_runs. Причина: зомби-worker оставил Redis-lock с TTL=2h, новые
задачи скипаются с reason='lock_held' молча.

Backend:
- _LOCK_TTL_SECONDS 7200 → 1800 (full sweep ~25-30 мин — потолок).
- force_release_lock() helper + новый POST /api/v1/admin/scrape/release-lock.
- TriggerKnRequest +force: bool — при true перед apply_async удалить lock.
- _log_task_received() / _log_task_skipped() — kn_scrape_log пишет «task
  подхвачен/skip» с run_id=NULL до основного flow. Теперь UI показывает
  whether worker реально получил задачу.

Frontend:
- Чекбокс ⚡ Force в форме запуска.
- Кнопка 🔓 Снять Redis-lock (отдельный endpoint).
- Уведомление о результате release.

.gitignore

@@ -13,6 +13,9 @@ node_modules/
 .next/
 frontend/.next/
 out/
+# TypeScript incremental build info — must NOT be committed; locally-generated
+# and changes on every build, which busts Docker build context cache.
+*.tsbuildinfo
 
 # Env / secrets
 .env

backend/.dockerignore

@@ -1,3 +1,7 @@
+# backend Dockerfile делает целевой `COPY app ./app` (не COPY .), поэтому
+# контекст и так чистый. Но build context передаётся ЦЕЛИКОМ от docker buildx
+# в daemon/builder — лишние мегабайты замедляют upload и хешинг.
+
 .venv
 __pycache__
 *.pyc
@@ -8,3 +12,28 @@ tests
 .env
 .env.*
 !.env.example
+
+# VCS
+.git
+.gitignore
+.gitattributes
+
+# Local data (raw downloads — никогда не нужны в образе)
+data/raw
+data/raw_endpoints
+data/photos
+
+# Logs / coverage
+*.log
+.coverage
+htmlcov
+
+# Docs / IDE
+*.md
+docs
+.vscode
+.idea
+
+# OS
+.DS_Store
+Thumbs.db

backend/Dockerfile

@@ -15,20 +15,23 @@ ENV PYTHONUNBUFFERED=1 \
     UV_COMPILE_BYTECODE=1 \
     UV_LINK_MODE=copy
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update && apt-get install -y --no-install-recommends \
     build-essential \
     libpq-dev \
     libgdal-dev \
     libproj-dev \
-    libgeos-dev \
+    libgeos-dev
-    && rm -rf /var/lib/apt/lists/*
 
 RUN pip install --no-cache-dir uv
 
 WORKDIR /app
 
 COPY pyproject.toml uv.lock* ./
-RUN if [ -f uv.lock ]; then uv sync --frozen --no-dev; else uv sync --no-dev; fi
+# uv-кеш переживает между билдами: при неизменном lock'е sync ~5 сек вместо 1-2 мин.
+RUN --mount=type=cache,target=/root/.cache/uv \
+    if [ -f uv.lock ]; then uv sync --frozen --no-dev; else uv sync --no-dev; fi
 
 COPY app ./app
 COPY alembic.ini ./
@@ -42,7 +45,9 @@ ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
     PATH="/app/.venv/bin:$PATH"
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update && apt-get install -y --no-install-recommends \
     libpq5 \
     libgdal36 \
     libproj25 \
@@ -52,7 +57,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libpangoft2-1.0-0 \
     fonts-dejavu-core \
     curl \
-    && rm -rf /var/lib/apt/lists/* \
     && useradd --create-home --uid 1000 app
 
 WORKDIR /app
@@ -75,7 +79,9 @@ FROM runner AS runner-with-chromium
 
 USER root
 
-RUN apt-get update && apt-get install -y --no-install-recommends \
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    apt-get update && apt-get install -y --no-install-recommends \
     # Chromium runtime libs (раньше заходили через `playwright install --with-deps`)
     libnss3 \
     libnspr4 \
@@ -97,7 +103,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 ENV PLAYWRIGHT_BROWSERS_PATH=/opt/playwright
 RUN mkdir -p $PLAYWRIGHT_BROWSERS_PATH \
     && /app/.venv/bin/playwright install chromium \
-    && chmod -R a+rX $PLAYWRIGHT_BROWSERS_PATH \
+    && chmod -R a+rX $PLAYWRIGHT_BROWSERS_PATH
-    && rm -rf /var/lib/apt/lists/*
 
 USER app

backend/app/api/v1/admin_scrape.py

@@ -29,6 +29,9 @@ class TriggerKnRequest(BaseModel):
     fetch_flats: bool = True
     extras: bool = True
     download_photos: bool = False
+    # Если True — ПЕРЕД apply_async удалить existing Redis lock. Эмердженси-режим
+    # для случая «зомби-worker оставил lock и блокирует новые задачи».
+    force: bool = False
 
 
 def _check_token(x_admin_token: str | None) -> None:
@@ -48,7 +51,14 @@ def trigger_kn_sweep(
 ) -> dict[str, Any]:
     _check_token(x_admin_token)
     # Defer Celery import so the API works without a broker in dev.
-    from app.workers.tasks.scrape_kn import scrape_kn_region
+    from app.workers.tasks.scrape_kn import (
+        force_release_lock,
+        scrape_kn_region,
+    )
+
+    lock_released = False
+    if payload.force:
+        lock_released = force_release_lock(payload.region_code, payload.developers)
 
     result = scrape_kn_region.apply_async(
         args=[payload.region_code, payload.developers],
@@ -64,6 +74,8 @@ def trigger_kn_sweep(
         "region_code": payload.region_code,
         "developers": payload.developers,
         "queued_at": result.date_done.isoformat() if result.date_done else None,
+        "force": payload.force,
+        "lock_was_released": lock_released,
     }
 
 
@@ -208,6 +220,29 @@ def queue_status(
     }
 
 
+class ReleaseLockRequest(BaseModel):
+    region_code: int = Field(..., ge=1, le=99)
+    developers: list[str] | None = Field(default=None, max_length=20)
+
+
+@router.post("/release-lock")
+def release_lock(
+    payload: ReleaseLockRequest,
+    x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
+) -> dict[str, Any]:
+    """Принудительно удалить Redis-lock для (region, devs). Использовать когда
+    зомби-worker оставил lock и новые задачи скипаются с reason='lock_held'."""
+    _check_token(x_admin_token)
+    from app.workers.tasks.scrape_kn import force_release_lock
+
+    released = force_release_lock(payload.region_code, payload.developers)
+    return {
+        "region_code": payload.region_code,
+        "developers": payload.developers,
+        "released": released,
+    }
+
+
 class RevokeRequest(BaseModel):
     task_id: str = Field(..., min_length=1, max_length=255)
     terminate: bool = False  # SIGTERM running task vs just unmark from queue

backend/app/workers/tasks/scrape_kn.py

@@ -18,18 +18,94 @@ from app.workers.celery_app import celery_app
 
 logger = logging.getLogger(__name__)
 
-# Lock TTL longer than the longest expected sweep (442-obj sweep with extras
+# Lock TTL — full Свердл sweep с extras занимает ~25-30 мин. 30 мин — потолок.
-# can run ~30-60 min). 2 hours is a safe upper bound — after that we assume
+# Если worker умер с lock'ом — он истечёт за полчаса, а не за 2 часа (раньше).
-# the worker died and let a new run proceed.
+# Heartbeat-based autoclean в /queue endpoint всё равно подхватит зомби-runs.
-_LOCK_TTL_SECONDS = 2 * 60 * 60
+_LOCK_TTL_SECONDS = 30 * 60
+
+
+def _lock_key(region_code: int, developers: list[str] | None) -> str:
+    devs_key = ",".join(developers) if developers else "*"
+    return f"scrape:kn:lock:{region_code}:{devs_key}"
+
+
+def _log_task_received(region_code: int, devs: list[str] | None, task_id: str | None) -> None:
+    """Записать в kn_scrape_log что worker получил задачу. Без run_id —
+    он появится позже в run_region_sweep. Best-effort: SQL ошибки гасим."""
+    from sqlalchemy import text
+
+    from app.core.db import SessionLocal
+
+    db = SessionLocal()
+    try:
+        db.execute(
+            text(
+                """
+                INSERT INTO kn_scrape_log (run_id, level, stage, message)
+                VALUES (NULL, 'info', 'task_received', :msg)
+                """
+            ),
+            {
+                "msg": (
+                    f"Worker подхватил task_id={task_id or '?'}, region={region_code},"
+                    f" devs={devs or '*'}"
+                )[:500],
+            },
+        )
+        db.commit()
+    except Exception as e:
+        logger.warning("log_task_received failed: %s", e)
+    finally:
+        db.close()
+
+
+def _log_task_skipped(
+    region_code: int, devs: list[str] | None, task_id: str | None, reason: str
+) -> None:
+    from sqlalchemy import text
+
+    from app.core.db import SessionLocal
+
+    db = SessionLocal()
+    try:
+        db.execute(
+            text(
+                """
+                INSERT INTO kn_scrape_log (run_id, level, stage, message)
+                VALUES (NULL, 'warn', 'task_skipped', :msg)
+                """
+            ),
+            {
+                "msg": (
+                    f"task_id={task_id or '?'} skipped: {reason}, region={region_code},"
+                    f" devs={devs or '*'}"
+                )[:500],
+            },
+        )
+        db.commit()
+    except Exception as e:
+        logger.warning("log_task_skipped failed: %s", e)
+    finally:
+        db.close()
+
+
+def force_release_lock(region_code: int, developers: list[str] | None) -> bool:
+    """Принудительно удалить Redis-lock (для emergency manual trigger,
+    когда зомби-worker оставил key). Возвращает True если ключ был."""
+    key = _lock_key(region_code, developers)
+    r = redis.Redis.from_url(settings.redis_url)
+    try:
+        return bool(r.delete(key))
+    except Exception as e:
+        logger.warning("force_release_lock %s failed: %s", key, e)
+        return False
 
 
 @contextmanager
 def _region_lock(region_code: int, developers: list[str] | None):
     """Redis singleton lock so beat double-fire or two workers can't run the
     same (region, devs) sweep concurrently. SET NX EX is atomic."""
-    devs_key = ",".join(developers) if developers else "*"
+    key = _lock_key(region_code, developers)
-    key = f"scrape:kn:lock:{region_code}:{devs_key}"
     r = redis.Redis.from_url(settings.redis_url)
     acquired = r.set(key, "1", nx=True, ex=_LOCK_TTL_SECONDS)
     try:
@@ -59,6 +135,12 @@ def scrape_kn_region(
     DOM.РФ at exactly the same minute. Manual triggers pass skip_jitter=True for
     immediate execution.
     """
+    # Сразу пишем «task received» в kn_scrape_log БЕЗ run_id (run_id ещё не
+    # создан — он появится в run_region_sweep). Это позволяет UI видеть
+    # «worker подхватил задачу» даже если потом сработает lock_held или
+    # упадём ещё до Phase A.
+    _log_task_received(region_code, developers, self.request.id)
+
     if not skip_jitter and settings.scrape_kn_jitter_seconds > 0:
         delay = random.uniform(0, settings.scrape_kn_jitter_seconds)
         logger.info("jitter delay %.0fs before scrape", delay)
@@ -77,11 +159,13 @@ def scrape_kn_region(
                 region_code,
                 developers,
             )
+            _log_task_skipped(region_code, developers, self.request.id, "lock_held")
             return {
                 "skipped": True,
                 "reason": "lock_held",
                 "region_code": region_code,
                 "developers": developers,
+                "lock_key": _lock_key(region_code, developers),
             }
 
         logger.info(
@@ -145,6 +229,8 @@ def resume_kn_run(self: Any, run_id: int) -> dict[str, Any]:
         return {"skipped": True, "reason": "no_region", "run_id": run_id}
     region_code = region_codes[0]
 
+    _log_task_received(region_code, developers, self.request.id)
+
     with _region_lock(region_code, developers) as got_lock:
         if not got_lock:
             logger.warning(
@@ -152,6 +238,7 @@ def resume_kn_run(self: Any, run_id: int) -> dict[str, Any]:
                 run_id,
                 region_code,
             )
+            _log_task_skipped(region_code, developers, self.request.id, "lock_held")
             return {"skipped": True, "reason": "lock_held", "run_id": run_id}
 
         logger.info("resume_kn_run start run=%s region=%s devs=%s", run_id, region_code, developers)

frontend/.dockerignore

@@ -1,6 +1,35 @@
+# Каждое лишнее изменение в build context инвалидирует layer cache в Docker.
+# Здесь — только то, что НЕ нужно для `npm ci` + `next build`.
+
 node_modules
 .next
+.next/cache
 .env
 .env.*
 !.env.example
 .eslintcache
+*.tsbuildinfo
+
+# VCS
+.git
+.gitignore
+.gitattributes
+
+# Logs / coverage / IDE
+*.log
+coverage
+.vscode
+.idea
+
+# Tests + docs (не нужны для production build)
+__tests__
+**/*.test.ts
+**/*.test.tsx
+**/*.spec.ts
+**/*.spec.tsx
+*.md
+docs
+
+# OS
+.DS_Store
+Thumbs.db

frontend/src/app/admin/scrape/page.tsx

@@ -76,6 +76,7 @@ export default function ScrapeAdminPage() {
   const [fetchFlats, setFetchFlats] = useState(true);
   const [extras, setExtras] = useState(true);
   const [downloadPhotos, setDownloadPhotos] = useState(false);
+  const [forceRun, setForceRun] = useState(false);
   const [showFailures, setShowFailures] = useState(false);
   const [failuresRunId, setFailuresRunId] = useState<number | "">("");
   const [logsRunId, setLogsRunId] = useState<number | "">("");
@@ -107,6 +108,7 @@ export default function ScrapeAdminPage() {
           fetch_flats: fetchFlats,
           extras,
           download_photos: downloadPhotos,
+          force: forceRun,
         }),
       }),
     onSuccess: (data) => {
@@ -118,6 +120,35 @@ export default function ScrapeAdminPage() {
     },
   });
 
+  const releaseLock = useMutation({
+    mutationFn: () =>
+      apiFetch<{
+        region_code: number;
+        developers: string[] | null;
+        released: boolean;
+      }>("/api/v1/admin/scrape/release-lock", {
+        method: "POST",
+        headers: { "X-Admin-Token": token },
+        body: JSON.stringify({
+          region_code: region,
+          developers: devs
+            ? devs
+                .split(",")
+                .map((s) => s.trim())
+                .filter(Boolean)
+            : null,
+        }),
+      }),
+    onSuccess: (data) => {
+      console.log("[scrape] lock released:", data);
+      queryClient.invalidateQueries({ queryKey: ["scrape-queue"] });
+    },
+    onError: (err) => {
+      console.error("[scrape] release-lock failed:", err);
+      alert(`Не удалось снять lock: ${String(err)}`);
+    },
+  });
+
   const queue = useQuery({
     queryKey: ["scrape-queue", token],
     queryFn: () =>
@@ -297,8 +328,28 @@ export default function ScrapeAdminPage() {
             />
             <span>Скачать фото (binary)</span>
           </label>
+          <label
+            style={{
+              display: "flex",
+              alignItems: "center",
+              gap: 6,
+              marginTop: 22,
+            }}
+            title="Перед запуском удалить existing Redis-lock (на случай зомби-worker'а)"
+          >
+            <input
+              type="checkbox"
+              checked={forceRun}
+              onChange={(e) => setForceRun(e.target.checked)}
+            />
+            <span style={{ color: forceRun ? "#b3261e" : "inherit" }}>
+              ⚡ Force (снять lock)
+            </span>
+          </label>
         </div>
-        <div style={{ marginTop: 16 }}>
+        <div
+          style={{ marginTop: 16, display: "flex", gap: 12, flexWrap: "wrap" }}
+        >
           <button
             onClick={() => trigger.mutate()}
             disabled={!token || trigger.isPending}
@@ -307,12 +358,53 @@ export default function ScrapeAdminPage() {
           >
             {trigger.isPending ? "Запускаю…" : "Запустить сейчас"}
           </button>
+          <button
+            onClick={() => {
+              if (
+                confirm(
+                  "Снять Redis-lock для этого региона/devs? Используй когда подозреваешь зомби-worker.",
+                )
+              ) {
+                releaseLock.mutate();
+              }
+            }}
+            disabled={!token || releaseLock.isPending}
+            style={secondaryBtn}
+            title="Удалить scrape:kn:lock:<region>:<devs> в Redis"
+          >
+            {releaseLock.isPending ? "Снимаю…" : "🔓 Снять Redis-lock"}
+          </button>
           {!token ? (
-            <span style={{ marginLeft: 12, color: "#9a6700", fontSize: 13 }}>
+            <span
+              style={{
+                marginLeft: 12,
+                color: "#9a6700",
+                fontSize: 13,
+                alignSelf: "center",
+              }}
+            >
               ← введите Admin Token выше
             </span>
           ) : null}
         </div>
+        {releaseLock.isSuccess ? (
+          <div
+            style={{
+              marginTop: 8,
+              padding: 8,
+              background: releaseLock.data.released ? "#f0fdf4" : "#f6f7f9",
+              border: "1px solid",
+              borderColor: releaseLock.data.released ? "#bbf7d0" : "#e6e8ec",
+              borderRadius: 6,
+              fontSize: 12,
+              color: releaseLock.data.released ? "#065f46" : "#5b6066",
+            }}
+          >
+            {releaseLock.data.released
+              ? `✓ Lock снят для region=${releaseLock.data.region_code} devs=${releaseLock.data.developers ?? "*"}. Можно запускать.`
+              : `Lock не был активен (region=${releaseLock.data.region_code}) — путь чист.`}
+          </div>
+        ) : null}
 
         {trigger.isError ? (
           <div