fix(scraper): диагностируем «кнопка не работает» — Redis lock + task_received log
Симптом: POST /admin/scrape/kn возвращает task_id, но run_id не появляется в kn_scrape_runs. Причина: зомби-worker оставил Redis-lock с TTL=2h, новые задачи скипаются с reason='lock_held' молча. Backend: - _LOCK_TTL_SECONDS 7200 → 1800 (full sweep ~25-30 мин — потолок). - force_release_lock() helper + новый POST /api/v1/admin/scrape/release-lock. - TriggerKnRequest +force: bool — при true перед apply_async удалить lock. - _log_task_received() / _log_task_skipped() — kn_scrape_log пишет «task подхвачен/skip» с run_id=NULL до основного flow. Теперь UI показывает whether worker реально получил задачу. Frontend: - Чекбокс ⚡ Force в форме запуска. - Кнопка 🔓 Снять Redis-lock (отдельный endpoint). - Уведомление о результате release.
This commit is contained in:
lekss361
parent
7a23aa9edc
commit
c3bc96535d
8 changed files with 298 additions and 19 deletions
3
.gitignore
vendored
3
.gitignore
vendored
|
|
@ -13,6 +13,9 @@ node_modules/
|
|||
.next/
|
||||
frontend/.next/
|
||||
out/
|
||||
# TypeScript incremental build info — must NOT be committed; locally-generated
|
||||
# and changes on every build, which busts Docker build context cache.
|
||||
*.tsbuildinfo
|
||||
|
||||
# Env / secrets
|
||||
.env
|
||||
|
|
|
|||
|
|
@ -1,3 +1,7 @@
|
|||
# backend Dockerfile делает целевой `COPY app ./app` (не COPY .), поэтому
|
||||
# контекст и так чистый. Но build context передаётся ЦЕЛИКОМ от docker buildx
|
||||
# в daemon/builder — лишние мегабайты замедляют upload и хешинг.
|
||||
|
||||
.venv
|
||||
__pycache__
|
||||
*.pyc
|
||||
|
|
@ -8,3 +12,28 @@ tests
|
|||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
|
||||
# VCS
|
||||
.git
|
||||
.gitignore
|
||||
.gitattributes
|
||||
|
||||
# Local data (raw downloads — никогда не нужны в образе)
|
||||
data/raw
|
||||
data/raw_endpoints
|
||||
data/photos
|
||||
|
||||
# Logs / coverage
|
||||
*.log
|
||||
.coverage
|
||||
htmlcov
|
||||
|
||||
# Docs / IDE
|
||||
*.md
|
||||
docs
|
||||
.vscode
|
||||
.idea
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
|
|
|||
|
|
@ -15,20 +15,23 @@ ENV PYTHONUNBUFFERED=1 \
|
|||
UV_COMPILE_BYTECODE=1 \
|
||||
UV_LINK_MODE=copy
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
libpq-dev \
|
||||
libgdal-dev \
|
||||
libproj-dev \
|
||||
libgeos-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
libgeos-dev
|
||||
|
||||
RUN pip install --no-cache-dir uv
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY pyproject.toml uv.lock* ./
|
||||
RUN if [ -f uv.lock ]; then uv sync --frozen --no-dev; else uv sync --no-dev; fi
|
||||
# uv-кеш переживает между билдами: при неизменном lock'е sync ~5 сек вместо 1-2 мин.
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
if [ -f uv.lock ]; then uv sync --frozen --no-dev; else uv sync --no-dev; fi
|
||||
|
||||
COPY app ./app
|
||||
COPY alembic.ini ./
|
||||
|
|
@ -42,7 +45,9 @@ ENV PYTHONUNBUFFERED=1 \
|
|||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
PATH="/app/.venv/bin:$PATH"
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
apt-get update && apt-get install -y --no-install-recommends \
|
||||
libpq5 \
|
||||
libgdal36 \
|
||||
libproj25 \
|
||||
|
|
@ -52,7 +57,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||
libpangoft2-1.0-0 \
|
||||
fonts-dejavu-core \
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& useradd --create-home --uid 1000 app
|
||||
|
||||
WORKDIR /app
|
||||
|
|
@ -75,7 +79,9 @@ FROM runner AS runner-with-chromium
|
|||
|
||||
USER root
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
|
||||
--mount=type=cache,target=/var/lib/apt,sharing=locked \
|
||||
apt-get update && apt-get install -y --no-install-recommends \
|
||||
# Chromium runtime libs (раньше заходили через `playwright install --with-deps`)
|
||||
libnss3 \
|
||||
libnspr4 \
|
||||
|
|
@ -97,7 +103,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
|
|||
ENV PLAYWRIGHT_BROWSERS_PATH=/opt/playwright
|
||||
RUN mkdir -p $PLAYWRIGHT_BROWSERS_PATH \
|
||||
&& /app/.venv/bin/playwright install chromium \
|
||||
&& chmod -R a+rX $PLAYWRIGHT_BROWSERS_PATH \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& chmod -R a+rX $PLAYWRIGHT_BROWSERS_PATH
|
||||
|
||||
USER app
|
||||
|
|
|
|||
|
|
@ -29,6 +29,9 @@ class TriggerKnRequest(BaseModel):
|
|||
fetch_flats: bool = True
|
||||
extras: bool = True
|
||||
download_photos: bool = False
|
||||
# Если True — ПЕРЕД apply_async удалить existing Redis lock. Эмердженси-режим
|
||||
# для случая «зомби-worker оставил lock и блокирует новые задачи».
|
||||
force: bool = False
|
||||
|
||||
|
||||
def _check_token(x_admin_token: str | None) -> None:
|
||||
|
|
@ -48,7 +51,14 @@ def trigger_kn_sweep(
|
|||
) -> dict[str, Any]:
|
||||
_check_token(x_admin_token)
|
||||
# Defer Celery import so the API works without a broker in dev.
|
||||
from app.workers.tasks.scrape_kn import scrape_kn_region
|
||||
from app.workers.tasks.scrape_kn import (
|
||||
force_release_lock,
|
||||
scrape_kn_region,
|
||||
)
|
||||
|
||||
lock_released = False
|
||||
if payload.force:
|
||||
lock_released = force_release_lock(payload.region_code, payload.developers)
|
||||
|
||||
result = scrape_kn_region.apply_async(
|
||||
args=[payload.region_code, payload.developers],
|
||||
|
|
@ -64,6 +74,8 @@ def trigger_kn_sweep(
|
|||
"region_code": payload.region_code,
|
||||
"developers": payload.developers,
|
||||
"queued_at": result.date_done.isoformat() if result.date_done else None,
|
||||
"force": payload.force,
|
||||
"lock_was_released": lock_released,
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -208,6 +220,29 @@ def queue_status(
|
|||
}
|
||||
|
||||
|
||||
class ReleaseLockRequest(BaseModel):
|
||||
region_code: int = Field(..., ge=1, le=99)
|
||||
developers: list[str] | None = Field(default=None, max_length=20)
|
||||
|
||||
|
||||
@router.post("/release-lock")
|
||||
def release_lock(
|
||||
payload: ReleaseLockRequest,
|
||||
x_admin_token: Annotated[str | None, Header(alias="X-Admin-Token")] = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Принудительно удалить Redis-lock для (region, devs). Использовать когда
|
||||
зомби-worker оставил lock и новые задачи скипаются с reason='lock_held'."""
|
||||
_check_token(x_admin_token)
|
||||
from app.workers.tasks.scrape_kn import force_release_lock
|
||||
|
||||
released = force_release_lock(payload.region_code, payload.developers)
|
||||
return {
|
||||
"region_code": payload.region_code,
|
||||
"developers": payload.developers,
|
||||
"released": released,
|
||||
}
|
||||
|
||||
|
||||
class RevokeRequest(BaseModel):
|
||||
task_id: str = Field(..., min_length=1, max_length=255)
|
||||
terminate: bool = False # SIGTERM running task vs just unmark from queue
|
||||
|
|
|
|||
|
|
@ -18,18 +18,94 @@ from app.workers.celery_app import celery_app
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Lock TTL longer than the longest expected sweep (442-obj sweep with extras
|
||||
# can run ~30-60 min). 2 hours is a safe upper bound — after that we assume
|
||||
# the worker died and let a new run proceed.
|
||||
_LOCK_TTL_SECONDS = 2 * 60 * 60
|
||||
# Lock TTL — full Свердл sweep с extras занимает ~25-30 мин. 30 мин — потолок.
|
||||
# Если worker умер с lock'ом — он истечёт за полчаса, а не за 2 часа (раньше).
|
||||
# Heartbeat-based autoclean в /queue endpoint всё равно подхватит зомби-runs.
|
||||
_LOCK_TTL_SECONDS = 30 * 60
|
||||
|
||||
|
||||
def _lock_key(region_code: int, developers: list[str] | None) -> str:
|
||||
devs_key = ",".join(developers) if developers else "*"
|
||||
return f"scrape:kn:lock:{region_code}:{devs_key}"
|
||||
|
||||
|
||||
def _log_task_received(region_code: int, devs: list[str] | None, task_id: str | None) -> None:
|
||||
"""Записать в kn_scrape_log что worker получил задачу. Без run_id —
|
||||
он появится позже в run_region_sweep. Best-effort: SQL ошибки гасим."""
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO kn_scrape_log (run_id, level, stage, message)
|
||||
VALUES (NULL, 'info', 'task_received', :msg)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"msg": (
|
||||
f"Worker подхватил task_id={task_id or '?'}, region={region_code},"
|
||||
f" devs={devs or '*'}"
|
||||
)[:500],
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
logger.warning("log_task_received failed: %s", e)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def _log_task_skipped(
|
||||
region_code: int, devs: list[str] | None, task_id: str | None, reason: str
|
||||
) -> None:
|
||||
from sqlalchemy import text
|
||||
|
||||
from app.core.db import SessionLocal
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
db.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO kn_scrape_log (run_id, level, stage, message)
|
||||
VALUES (NULL, 'warn', 'task_skipped', :msg)
|
||||
"""
|
||||
),
|
||||
{
|
||||
"msg": (
|
||||
f"task_id={task_id or '?'} skipped: {reason}, region={region_code},"
|
||||
f" devs={devs or '*'}"
|
||||
)[:500],
|
||||
},
|
||||
)
|
||||
db.commit()
|
||||
except Exception as e:
|
||||
logger.warning("log_task_skipped failed: %s", e)
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
|
||||
def force_release_lock(region_code: int, developers: list[str] | None) -> bool:
|
||||
"""Принудительно удалить Redis-lock (для emergency manual trigger,
|
||||
когда зомби-worker оставил key). Возвращает True если ключ был."""
|
||||
key = _lock_key(region_code, developers)
|
||||
r = redis.Redis.from_url(settings.redis_url)
|
||||
try:
|
||||
return bool(r.delete(key))
|
||||
except Exception as e:
|
||||
logger.warning("force_release_lock %s failed: %s", key, e)
|
||||
return False
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _region_lock(region_code: int, developers: list[str] | None):
|
||||
"""Redis singleton lock so beat double-fire or two workers can't run the
|
||||
same (region, devs) sweep concurrently. SET NX EX is atomic."""
|
||||
devs_key = ",".join(developers) if developers else "*"
|
||||
key = f"scrape:kn:lock:{region_code}:{devs_key}"
|
||||
key = _lock_key(region_code, developers)
|
||||
r = redis.Redis.from_url(settings.redis_url)
|
||||
acquired = r.set(key, "1", nx=True, ex=_LOCK_TTL_SECONDS)
|
||||
try:
|
||||
|
|
@ -59,6 +135,12 @@ def scrape_kn_region(
|
|||
DOM.РФ at exactly the same minute. Manual triggers pass skip_jitter=True for
|
||||
immediate execution.
|
||||
"""
|
||||
# Сразу пишем «task received» в kn_scrape_log БЕЗ run_id (run_id ещё не
|
||||
# создан — он появится в run_region_sweep). Это позволяет UI видеть
|
||||
# «worker подхватил задачу» даже если потом сработает lock_held или
|
||||
# упадём ещё до Phase A.
|
||||
_log_task_received(region_code, developers, self.request.id)
|
||||
|
||||
if not skip_jitter and settings.scrape_kn_jitter_seconds > 0:
|
||||
delay = random.uniform(0, settings.scrape_kn_jitter_seconds)
|
||||
logger.info("jitter delay %.0fs before scrape", delay)
|
||||
|
|
@ -77,11 +159,13 @@ def scrape_kn_region(
|
|||
region_code,
|
||||
developers,
|
||||
)
|
||||
_log_task_skipped(region_code, developers, self.request.id, "lock_held")
|
||||
return {
|
||||
"skipped": True,
|
||||
"reason": "lock_held",
|
||||
"region_code": region_code,
|
||||
"developers": developers,
|
||||
"lock_key": _lock_key(region_code, developers),
|
||||
}
|
||||
|
||||
logger.info(
|
||||
|
|
@ -145,6 +229,8 @@ def resume_kn_run(self: Any, run_id: int) -> dict[str, Any]:
|
|||
return {"skipped": True, "reason": "no_region", "run_id": run_id}
|
||||
region_code = region_codes[0]
|
||||
|
||||
_log_task_received(region_code, developers, self.request.id)
|
||||
|
||||
with _region_lock(region_code, developers) as got_lock:
|
||||
if not got_lock:
|
||||
logger.warning(
|
||||
|
|
@ -152,6 +238,7 @@ def resume_kn_run(self: Any, run_id: int) -> dict[str, Any]:
|
|||
run_id,
|
||||
region_code,
|
||||
)
|
||||
_log_task_skipped(region_code, developers, self.request.id, "lock_held")
|
||||
return {"skipped": True, "reason": "lock_held", "run_id": run_id}
|
||||
|
||||
logger.info("resume_kn_run start run=%s region=%s devs=%s", run_id, region_code, developers)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,35 @@
|
|||
# Каждое лишнее изменение в build context инвалидирует layer cache в Docker.
|
||||
# Здесь — только то, что НЕ нужно для `npm ci` + `next build`.
|
||||
|
||||
node_modules
|
||||
.next
|
||||
.next/cache
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
.eslintcache
|
||||
*.tsbuildinfo
|
||||
|
||||
# VCS
|
||||
.git
|
||||
.gitignore
|
||||
.gitattributes
|
||||
|
||||
# Logs / coverage / IDE
|
||||
*.log
|
||||
coverage
|
||||
.vscode
|
||||
.idea
|
||||
|
||||
# Tests + docs (не нужны для production build)
|
||||
__tests__
|
||||
**/*.test.ts
|
||||
**/*.test.tsx
|
||||
**/*.spec.ts
|
||||
**/*.spec.tsx
|
||||
*.md
|
||||
docs
|
||||
|
||||
# OS
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
|
|
|
|||
|
|
@ -76,6 +76,7 @@ export default function ScrapeAdminPage() {
|
|||
const [fetchFlats, setFetchFlats] = useState(true);
|
||||
const [extras, setExtras] = useState(true);
|
||||
const [downloadPhotos, setDownloadPhotos] = useState(false);
|
||||
const [forceRun, setForceRun] = useState(false);
|
||||
const [showFailures, setShowFailures] = useState(false);
|
||||
const [failuresRunId, setFailuresRunId] = useState<number | "">("");
|
||||
const [logsRunId, setLogsRunId] = useState<number | "">("");
|
||||
|
|
@ -107,6 +108,7 @@ export default function ScrapeAdminPage() {
|
|||
fetch_flats: fetchFlats,
|
||||
extras,
|
||||
download_photos: downloadPhotos,
|
||||
force: forceRun,
|
||||
}),
|
||||
}),
|
||||
onSuccess: (data) => {
|
||||
|
|
@ -118,6 +120,35 @@ export default function ScrapeAdminPage() {
|
|||
},
|
||||
});
|
||||
|
||||
const releaseLock = useMutation({
|
||||
mutationFn: () =>
|
||||
apiFetch<{
|
||||
region_code: number;
|
||||
developers: string[] | null;
|
||||
released: boolean;
|
||||
}>("/api/v1/admin/scrape/release-lock", {
|
||||
method: "POST",
|
||||
headers: { "X-Admin-Token": token },
|
||||
body: JSON.stringify({
|
||||
region_code: region,
|
||||
developers: devs
|
||||
? devs
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)
|
||||
: null,
|
||||
}),
|
||||
}),
|
||||
onSuccess: (data) => {
|
||||
console.log("[scrape] lock released:", data);
|
||||
queryClient.invalidateQueries({ queryKey: ["scrape-queue"] });
|
||||
},
|
||||
onError: (err) => {
|
||||
console.error("[scrape] release-lock failed:", err);
|
||||
alert(`Не удалось снять lock: ${String(err)}`);
|
||||
},
|
||||
});
|
||||
|
||||
const queue = useQuery({
|
||||
queryKey: ["scrape-queue", token],
|
||||
queryFn: () =>
|
||||
|
|
@ -297,8 +328,28 @@ export default function ScrapeAdminPage() {
|
|||
/>
|
||||
<span>Скачать фото (binary)</span>
|
||||
</label>
|
||||
<label
|
||||
style={{
|
||||
display: "flex",
|
||||
alignItems: "center",
|
||||
gap: 6,
|
||||
marginTop: 22,
|
||||
}}
|
||||
title="Перед запуском удалить existing Redis-lock (на случай зомби-worker'а)"
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={forceRun}
|
||||
onChange={(e) => setForceRun(e.target.checked)}
|
||||
/>
|
||||
<span style={{ color: forceRun ? "#b3261e" : "inherit" }}>
|
||||
⚡ Force (снять lock)
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
<div style={{ marginTop: 16 }}>
|
||||
<div
|
||||
style={{ marginTop: 16, display: "flex", gap: 12, flexWrap: "wrap" }}
|
||||
>
|
||||
<button
|
||||
onClick={() => trigger.mutate()}
|
||||
disabled={!token || trigger.isPending}
|
||||
|
|
@ -307,12 +358,53 @@ export default function ScrapeAdminPage() {
|
|||
>
|
||||
{trigger.isPending ? "Запускаю…" : "Запустить сейчас"}
|
||||
</button>
|
||||
<button
|
||||
onClick={() => {
|
||||
if (
|
||||
confirm(
|
||||
"Снять Redis-lock для этого региона/devs? Используй когда подозреваешь зомби-worker.",
|
||||
)
|
||||
) {
|
||||
releaseLock.mutate();
|
||||
}
|
||||
}}
|
||||
disabled={!token || releaseLock.isPending}
|
||||
style={secondaryBtn}
|
||||
title="Удалить scrape:kn:lock:<region>:<devs> в Redis"
|
||||
>
|
||||
{releaseLock.isPending ? "Снимаю…" : "🔓 Снять Redis-lock"}
|
||||
</button>
|
||||
{!token ? (
|
||||
<span style={{ marginLeft: 12, color: "#9a6700", fontSize: 13 }}>
|
||||
<span
|
||||
style={{
|
||||
marginLeft: 12,
|
||||
color: "#9a6700",
|
||||
fontSize: 13,
|
||||
alignSelf: "center",
|
||||
}}
|
||||
>
|
||||
← введите Admin Token выше
|
||||
</span>
|
||||
) : null}
|
||||
</div>
|
||||
{releaseLock.isSuccess ? (
|
||||
<div
|
||||
style={{
|
||||
marginTop: 8,
|
||||
padding: 8,
|
||||
background: releaseLock.data.released ? "#f0fdf4" : "#f6f7f9",
|
||||
border: "1px solid",
|
||||
borderColor: releaseLock.data.released ? "#bbf7d0" : "#e6e8ec",
|
||||
borderRadius: 6,
|
||||
fontSize: 12,
|
||||
color: releaseLock.data.released ? "#065f46" : "#5b6066",
|
||||
}}
|
||||
>
|
||||
{releaseLock.data.released
|
||||
? `✓ Lock снят для region=${releaseLock.data.region_code} devs=${releaseLock.data.developers ?? "*"}. Можно запускать.`
|
||||
: `Lock не был активен (region=${releaseLock.data.region_code}) — путь чист.`}
|
||||
</div>
|
||||
) : null}
|
||||
|
||||
{trigger.isError ? (
|
||||
<div
|
||||
|
|
|
|||
File diff suppressed because one or more lines are too long
Loading…
Add table
Reference in a new issue