8c9d1344cc
Root cause: `cat $ISSUES_TMP | python3 <<EOF ... sys.exit(0)` — Python exits 0 early, cat получает SIGPIPE, set -o pipefail → script exits 141 = failure. Fix: передача file path через env var вместо stdin pipe. Python читает файл напрямую. No pipe → no SIGPIPE. Verified локально на VPS с bot-qa token: exit 0, '[OK] 0 wip issues — nothing to do'. Also remove diagnostic steps from workflow (added в PR #611, нужно для поиска root cause, теперь убираем). Refs: PR #611 (diagnostic), runs #1514-1520 failures.
164 lines
6.2 KiB
Bash
164 lines
6.2 KiB
Bash
#!/usr/bin/env bash
|
||
# Cleanup stale claims for autonomous multi-agent workflow.
|
||
#
|
||
# Освобождает issues застрявшие в `status/wip` дольше 4 часов:
|
||
# - убирает assignee
|
||
# - меняет label status/wip → status/ready (delete wip → add ready)
|
||
#
|
||
# Why: если worker crashes или window закрывается во время работы, issue
|
||
# остаётся wip+assigned forever. Без cleanup queue забивается.
|
||
#
|
||
# Run: cron каждые 30 минут (см. .forgejo/workflows/stale-claims.yml).
|
||
#
|
||
# Env vars (требуются):
|
||
# FORGEJO_TOKEN — admin или с write:issue scope на repo
|
||
# FORGEJO_URL — https://git.gendsgn.ru (без /api/v1)
|
||
# FORGEJO_REPO — lekss361/gendesign
|
||
# STALE_HOURS — порог в часах (default 4)
|
||
#
|
||
# Assumption: issue.updated_at = последний touch (label/comment/assignee
|
||
# change). Worker делающий task >4h без касаний issue будет считаться stale.
|
||
# Для текущих small tasks 4h enough; в Phase 4 можно добавить heartbeat
|
||
# comment'ы от worker'а.
|
||
#
|
||
# Exit codes:
|
||
# 0 — success (even if 0 stale found)
|
||
# 1 — env vars missing / API error
|
||
|
||
set -euo pipefail
|
||
|
||
: "${FORGEJO_TOKEN:?FORGEJO_TOKEN required}"
|
||
: "${FORGEJO_URL:?FORGEJO_URL required}"
|
||
: "${FORGEJO_REPO:?FORGEJO_REPO required}"
|
||
|
||
export STALE_HOURS="${STALE_HOURS:-4}"
|
||
STALE_SECS=$((STALE_HOURS * 3600))
|
||
|
||
curl_forgejo() {
|
||
curl -sS -H "Authorization: token $FORGEJO_TOKEN" \
|
||
-H "Content-Type: application/json" \
|
||
"$FORGEJO_URL/api/v1/$1" "${@:2}"
|
||
}
|
||
|
||
now_utc_epoch() {
|
||
python3 -c 'import datetime; print(int(datetime.datetime.now(datetime.timezone.utc).timestamp()))'
|
||
}
|
||
|
||
echo "[$(date -Is)] cleanup-stale-claims: scanning status/wip issues (stale threshold: ${STALE_HOURS}h)"
|
||
|
||
NOW=$(now_utc_epoch)
|
||
export CUTOFF=$((NOW - STALE_SECS))
|
||
|
||
# GET all open issues с label status/wip — pagination в Python (более надёжно чем
|
||
# склеивать JSON arrays в bash). Defensive cap 10 pages × 50 = 500.
|
||
|
||
ISSUES_TMP=$(mktemp)
|
||
trap 'rm -f "$ISSUES_TMP"' EXIT
|
||
|
||
for PAGE in 1 2 3 4 5 6 7 8 9 10; do
|
||
PAGE_RESP=$(curl_forgejo "repos/$FORGEJO_REPO/issues?labels=status/wip&state=open&limit=50&page=$PAGE")
|
||
echo "$PAGE_RESP" >> "$ISSUES_TMP"
|
||
echo "---PAGE_END---" >> "$ISSUES_TMP"
|
||
PAGE_LEN=$(echo "$PAGE_RESP" | python3 -c 'import json,sys; print(len(json.load(sys.stdin)))')
|
||
if [ "$PAGE_LEN" -lt 50 ]; then
|
||
break
|
||
fi
|
||
done
|
||
|
||
# Парсим JSON — читаем файл прямо в Python через env var (не через pipe `cat | python3`).
|
||
# При pipe + sys.exit(0) после print, cat получает SIGPIPE → script exits 141 под
|
||
# set -o pipefail. Передача path через env var не использует pipe → no SIGPIPE.
|
||
ISSUES_FILE="$ISSUES_TMP" python3 <<'PYEOF'
|
||
import json, os, subprocess, sys, datetime
|
||
|
||
cutoff = int(os.environ.get("CUTOFF", "0"))
|
||
if cutoff == 0:
|
||
print("[ERROR] CUTOFF env var не выставлен или равен 0 — abort", file=sys.stderr)
|
||
sys.exit(1)
|
||
|
||
# Read pagination output — multiple JSON arrays separated by "---PAGE_END---"
|
||
with open(os.environ["ISSUES_FILE"]) as f:
|
||
raw = f.read()
|
||
pages = [p.strip() for p in raw.split("---PAGE_END---") if p.strip()]
|
||
data = []
|
||
for p in pages:
|
||
try:
|
||
data.extend(json.loads(p))
|
||
except json.JSONDecodeError:
|
||
print(f"[WARN] failed to parse page (len={len(p)}), skipping", file=sys.stderr)
|
||
|
||
if not data:
|
||
print("[OK] 0 wip issues — nothing to do")
|
||
sys.exit(0)
|
||
|
||
released = 0
|
||
errors = 0
|
||
now_epoch = int(datetime.datetime.now(datetime.timezone.utc).timestamp())
|
||
|
||
for issue in data:
|
||
issue_num = issue["number"]
|
||
updated_at = issue["updated_at"]
|
||
title = issue["title"]
|
||
assignees = [a["login"] for a in (issue.get("assignees") or [])]
|
||
|
||
updated_epoch = int(datetime.datetime.fromisoformat(updated_at.replace("Z", "+00:00")).timestamp())
|
||
age_secs = now_epoch - updated_epoch
|
||
age_hours = age_secs // 3600
|
||
|
||
if updated_epoch >= cutoff:
|
||
print(f" · #{issue_num} '{title[:50]}' — age {age_hours}h, fresh, skip")
|
||
continue
|
||
|
||
print(f" ⚠ #{issue_num} '{title[:50]}' — age {age_hours}h, STALE")
|
||
print(f" assignees: {assignees}")
|
||
print(f" → releasing claim")
|
||
|
||
repo = os.environ["FORGEJO_REPO"]
|
||
base = os.environ["FORGEJO_URL"] + "/api/v1"
|
||
token = os.environ["FORGEJO_TOKEN"]
|
||
auth = ["-H", f"Authorization: token {token}", "-H", "Content-Type: application/json"]
|
||
|
||
try:
|
||
# 1. Clear assignees
|
||
subprocess.run(
|
||
["curl", "-sS", "-X", "PATCH", *auth, "-d", '{"assignees": []}',
|
||
f"{base}/repos/{repo}/issues/{issue_num}"],
|
||
check=True, capture_output=True,
|
||
)
|
||
|
||
# 2. Remove status/wip label FIRST (если падает — issue в neutral state без status/*,
|
||
# что менее опасно чем оба status/ready+status/wip одновременно)
|
||
subprocess.run(
|
||
["curl", "-sS", "-X", "DELETE", *auth,
|
||
f"{base}/repos/{repo}/issues/{issue_num}/labels/status%2Fwip"],
|
||
check=True, capture_output=True,
|
||
)
|
||
|
||
# 3. Add status/ready label
|
||
subprocess.run(
|
||
["curl", "-sS", "-X", "POST", *auth, "-d", '{"labels": ["status/ready"]}',
|
||
f"{base}/repos/{repo}/issues/{issue_num}/labels"],
|
||
check=True, capture_output=True,
|
||
)
|
||
|
||
# 4. Post comment for trace
|
||
comment_body = json.dumps({
|
||
"body": f"⚠️ Stale claim released: issue stuck in status/wip for {age_hours}h "
|
||
f"(previously assigned to {', '.join(assignees) or 'none'}). "
|
||
f"Label transition: status/wip → status/ready. Worker likely crashed."
|
||
})
|
||
subprocess.run(
|
||
["curl", "-sS", "-X", "POST", *auth, "-d", comment_body,
|
||
f"{base}/repos/{repo}/issues/{issue_num}/comments"],
|
||
check=True, capture_output=True,
|
||
)
|
||
|
||
released += 1
|
||
except subprocess.CalledProcessError as e:
|
||
print(f" ✗ FAILED: {e.stderr.decode() if e.stderr else 'unknown'}")
|
||
errors += 1
|
||
|
||
print()
|
||
print(f"[OK] Released {released} stale claim(s), {errors} errors")
|
||
sys.exit(1 if errors else 0)
|
||
PYEOF
|