lekss361
f52601de41
All checks were successful
Deploy Infra Host / sync-infra-host (push) Successful in 2s
Deploy / changes (push) Successful in 8s
Deploy / build-backend (push) Has been skipped
Deploy / deploy-caddy (push) Has been skipped
Deploy / perimeter-smoke (push) Successful in 9s
Deploy / build-worker (push) Has been skipped
Deploy / build-frontend (push) Has been skipped
Deploy / deploy (push) Successful in 1m23s
Deploy / deploy-status (push) Successful in 1s
253 lines
13 KiB
Bash
Executable file
253 lines
13 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
# Backup for self-hosted Forgejo (git.gendsgn.ru) — repos + DB, off-box to S3
|
||
# under its OWN, more-restricted key (#2203).
|
||
#
|
||
# Forgejo is deployed OUTSIDE this repo's checkout, per
|
||
# infra/Forgejo_Migration_BotServer_To_Beget_2026-05-16.md (vault):
|
||
# - compose dir on the VM: /home/gendesign/forgejo/
|
||
# - repo data: /home/gendesign/forgejo/data/forgejo/git/repositories/
|
||
# - DB: NOT a dedicated container — a separate `forgejo` user+database
|
||
# inside the SAME shared postgres container as the main app
|
||
# (gendesign-postgres-1). `pg_dump -U forgejo forgejo` via `docker exec`.
|
||
#
|
||
# Two artifacts per run:
|
||
# 1. DB dump — `pg_dump --no-owner --clean --if-exists` (repos/issues/PRs/
|
||
# users/settings/Actions config — everything except the git data itself).
|
||
# 2. Repo bundle — a tar.gz of the bare-repo tree under FORGEJO_REPOS_DIR.
|
||
# This is functionally a "bundle of repositories": bare repos already
|
||
# ARE just refs+objects, byte-identical to what `git bundle` would
|
||
# capture, and tar-ing the whole tree in one shot is far more robust
|
||
# than enumerating individual repos (new repos need zero script changes,
|
||
# nothing to keep in sync). LFS objects/attachments/avatars under
|
||
# FORGEJO_DATA_DIR are OUT of scope for this pass — repo code + full DB
|
||
# (which has LFS pointers, not the blobs) covers the "backup the code"
|
||
# ask; note this gap in the PR if larger asset backup is wanted later.
|
||
#
|
||
# Off-box S3 upload is MANDATORY here, not optional like ops/backup.sh —
|
||
# the whole point of this script is getting Forgejo's code off the VM it
|
||
# lives on, so with no S3 creds configured this refuses to run rather than
|
||
# silently producing a local-only "backup" that doesn't meet that bar.
|
||
#
|
||
# The S3 key here is a SEPARATE, narrower-scoped service user than the one
|
||
# ops/backup.sh and backup-tradein-db.sh use (`gendsgn-backup-writer`, which
|
||
# can write anywhere in the bucket root) — this one can ONLY PutObject under
|
||
# s3://gendsgn-backups/forgejo/*, nothing else, so a compromised Forgejo host
|
||
# can't touch the main/tradein DB backups sitting in the same bucket. See the
|
||
# PR description for the exact bucket policy JSON (mirrors the existing
|
||
# gendsgn-backup-writer policy pattern — vault meta/00_credentials.md
|
||
# "Selectel S3 — бэкап-бакет gendsgn-backups").
|
||
#
|
||
# THE KEY DOES NOT EXIST YET (as of #2203) — a human needs to create the
|
||
# service user + policy in the Selectel panel first. Until then this script
|
||
# is expected to fail loudly and exit non-zero; that is the correct,
|
||
# intentional behaviour, not a bug.
|
||
#
|
||
# Missed-run detection (#2203): same sentinel/staleness-check pattern as
|
||
# ops/backup.sh — see ops/check-backup-staleness.sh for the alerting side.
|
||
#
|
||
# Usage (cron — `bash <path>`, not a bare path, same "+x can go missing on a
|
||
# raw deploy reset" reasoning as the other two backup scripts, #71):
|
||
# 15 4 * * * bash /opt/gendesign/ops/backup-forgejo.sh >> /var/log/gendesign-backup-forgejo.log 2>&1
|
||
#
|
||
# Config file (root-owned, chmod 600, NOT in git) — see
|
||
# ops/gendesign-backup-forgejo.default.example for the full list:
|
||
# FORGEJO_S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru
|
||
# FORGEJO_S3_BUCKET=gendsgn-backups
|
||
# FORGEJO_S3_PREFIX=forgejo/
|
||
# FORGEJO_S3_ACCESS_KEY=...
|
||
# FORGEJO_S3_SECRET_KEY=...
|
||
|
||
set -euo pipefail
|
||
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
# shellcheck source=./lib-backup.sh
|
||
source "$SCRIPT_DIR/lib-backup.sh"
|
||
|
||
# --- config (env-overridable) ---
|
||
FORGEJO_DIR="${FORGEJO_DIR:-/home/gendesign/forgejo}"
|
||
FORGEJO_DATA_DIR="${FORGEJO_DATA_DIR:-${FORGEJO_DIR}/data/forgejo}"
|
||
FORGEJO_REPOS_DIR="${FORGEJO_REPOS_DIR:-${FORGEJO_DATA_DIR}/git/repositories}"
|
||
|
||
PG_CONTAINER="${PG_CONTAINER:-gendesign-postgres-1}"
|
||
FORGEJO_DB_USER="${FORGEJO_DB_USER:-forgejo}"
|
||
FORGEJO_DB_NAME="${FORGEJO_DB_NAME:-forgejo}"
|
||
|
||
LOCAL_BACKUP_DIR="${LOCAL_BACKUP_DIR:-/opt/gendesign/backups/forgejo}"
|
||
KEEP="${KEEP:-7}"
|
||
MIN_DB_DUMP_BYTES="${MIN_DB_DUMP_BYTES:-2048}" # forgejo DB is small (config/issues/PRs, no git blobs)
|
||
MIN_REPOS_BUNDLE_BYTES="${MIN_REPOS_BUNDLE_BYTES:-10240}"
|
||
SENTINEL_FILE="${SENTINEL_FILE:-${LOCAL_BACKUP_DIR}/.last_success}"
|
||
|
||
FORGEJO_BACKUP_ENV_FILE="${FORGEJO_BACKUP_ENV_FILE:-/etc/default/gendesign-backup-forgejo}"
|
||
# shellcheck source=/dev/null
|
||
[[ -f "$FORGEJO_BACKUP_ENV_FILE" ]] && source "$FORGEJO_BACKUP_ENV_FILE"
|
||
|
||
FORGEJO_S3_ENDPOINT="${FORGEJO_S3_ENDPOINT:-}"
|
||
FORGEJO_S3_BUCKET="${FORGEJO_S3_BUCKET:-}"
|
||
FORGEJO_S3_PREFIX="${FORGEJO_S3_PREFIX:-forgejo/}"
|
||
FORGEJO_S3_ACCESS_KEY="${FORGEJO_S3_ACCESS_KEY:-}"
|
||
FORGEJO_S3_SECRET_KEY="${FORGEJO_S3_SECRET_KEY:-}"
|
||
|
||
# --- guard 0: S3 creds MUST be configured — this backup's whole point is
|
||
# off-box storage under a narrow-scoped key. No creds = refuse to run,
|
||
# loudly, rather than silently producing a local-only "backup" (unlike
|
||
# ops/backup.sh, where local-only is an accepted fallback). ---
|
||
if [[ -z "$FORGEJO_S3_ENDPOINT" || -z "$FORGEJO_S3_BUCKET" || -z "$FORGEJO_S3_ACCESS_KEY" || -z "$FORGEJO_S3_SECRET_KEY" ]]; then
|
||
log "ERROR: Forgejo S3 backup is NOT CONFIGURED."
|
||
log "ERROR: missing one or more of FORGEJO_S3_ENDPOINT / FORGEJO_S3_BUCKET / FORGEJO_S3_ACCESS_KEY / FORGEJO_S3_SECRET_KEY."
|
||
log "ERROR: create the narrow-scoped 'gendsgn-backup-forgejo-writer' S3 service user (see PR for the exact bucket policy)"
|
||
log "ERROR: and fill them into ${FORGEJO_BACKUP_ENV_FILE} (see ops/gendesign-backup-forgejo.default.example)."
|
||
log "ERROR: refusing to produce a local-only backup — that does not meet this script's off-box requirement."
|
||
exit 1
|
||
fi
|
||
|
||
if [[ ! -d "$FORGEJO_REPOS_DIR" ]]; then
|
||
log "ERROR: FORGEJO_REPOS_DIR does not exist -> $FORGEJO_REPOS_DIR"
|
||
log "ERROR: Forgejo's on-disk layout may differ from the documented default — override FORGEJO_REPOS_DIR."
|
||
exit 1
|
||
fi
|
||
|
||
mkdir -p "$LOCAL_BACKUP_DIR"
|
||
ts=$(date -u +'%Y%m%d_%H%M%S')
|
||
db_out="${LOCAL_BACKUP_DIR}/forgejo-db_${ts}.sql.gz"
|
||
repos_out="${LOCAL_BACKUP_DIR}/forgejo-repos_${ts}.tar.gz"
|
||
|
||
# --- 1. DB dump ---
|
||
log "Dumping Forgejo DB (${FORGEJO_DB_NAME} as ${FORGEJO_DB_USER}, container ${PG_CONTAINER}) -> ${db_out}"
|
||
docker exec "$PG_CONTAINER" pg_dump -U "$FORGEJO_DB_USER" -d "$FORGEJO_DB_NAME" --no-owner --clean --if-exists \
|
||
| gzip -9 > "$db_out"
|
||
|
||
if [[ ! -s "$db_out" ]]; then
|
||
log "ERROR: DB dump is empty -> $db_out — removing, keeping previous good backups." >&2
|
||
rm -f "$db_out"
|
||
exit 1
|
||
fi
|
||
|
||
db_bytes=$(wc -c < "$db_out" | tr -d ' ')
|
||
if (( db_bytes < MIN_DB_DUMP_BYTES )); then
|
||
log "ERROR: DB dump only ${db_bytes} bytes (< ${MIN_DB_DUMP_BYTES} floor) — likely a failed dump." >&2
|
||
rm -f "$db_out"
|
||
exit 1
|
||
fi
|
||
|
||
if ! verify_dump_integrity "$db_out" "-- PostgreSQL database dump complete" "forgejo DB dump"; then
|
||
log "Removing suspect DB dump, NOT pruning older good backups." >&2
|
||
rm -f "$db_out"
|
||
exit 1
|
||
fi
|
||
|
||
log "DB dump OK: ${db_out} ($(du -h "$db_out" | cut -f1), ${db_bytes} bytes)"
|
||
|
||
# --- 2. repo bundle (tar.gz of the bare-repo tree) ---
|
||
log "Bundling Forgejo repos from ${FORGEJO_REPOS_DIR} -> ${repos_out}"
|
||
tar -czf "$repos_out" -C "$(dirname "$FORGEJO_REPOS_DIR")" "$(basename "$FORGEJO_REPOS_DIR")"
|
||
|
||
if [[ ! -s "$repos_out" ]]; then
|
||
log "ERROR: repo bundle is empty -> $repos_out — removing." >&2
|
||
rm -f "$repos_out"
|
||
exit 1
|
||
fi
|
||
|
||
repos_bytes=$(wc -c < "$repos_out" | tr -d ' ')
|
||
if (( repos_bytes < MIN_REPOS_BUNDLE_BYTES )); then
|
||
log "ERROR: repo bundle only ${repos_bytes} bytes (< ${MIN_REPOS_BUNDLE_BYTES} floor) — likely empty/failed FORGEJO_REPOS_DIR." >&2
|
||
rm -f "$repos_out"
|
||
exit 1
|
||
fi
|
||
|
||
# tar integrity: a bundle truncated mid-write (disk full, killed process)
|
||
# fails `tar -t` even when the gzip framing looks superficially OK — same
|
||
# "don't trust non-empty + big-enough alone" discipline as verify_dump_integrity.
|
||
if ! tar -tzf "$repos_out" >/dev/null 2>&1; then
|
||
log "ERROR: repo bundle failed tar integrity check -> $repos_out (truncated archive?)" >&2
|
||
rm -f "$repos_out"
|
||
exit 1
|
||
fi
|
||
|
||
log "Repo bundle OK: ${repos_out} ($(du -h "$repos_out" | cut -f1), ${repos_bytes} bytes)"
|
||
|
||
# --- 3. config bundle (#2203 follow-up) ---
|
||
# ПОЧЕМУ ЭТО ОТДЕЛЬНЫЙ АРТЕФАКТ. Бэкап забирал БД и голые репозитории — то есть
|
||
# СОДЕРЖИМОЕ Forgejo, но не то, ЧЕМ оно поднимается. Компоуз-файлы Forgejo и
|
||
# раннеров живут в /home/gendesign/forgejo/, вне репозитория (проверено:
|
||
# `git rev-parse` там отвечает "not a git repository"), их не трогает ни один
|
||
# деплой и не покрывал ни один бэкап.
|
||
#
|
||
# Практический смысл: при потере хоста БД и репозитории восстанавливаются, а
|
||
# КОНФИГУРАЦИЯ — нет. Пришлось бы вручную воссоздавать docker-compose.yml и
|
||
# заново регистрировать три раннера. Файлы крошечные (~5 КБ на оба), так что
|
||
# цена включения нулевая, а цена отсутствия — ручная реконструкция в худший
|
||
# момент.
|
||
#
|
||
# Косвенное подтверждение, что каталог правится руками и без истории: рядом
|
||
# лежат runner-compose.yml.bak, .bak.1779049805, .bak.before-runner3 — три
|
||
# самодельных снимка вместо version control.
|
||
#
|
||
# ЧТО НАМЕРЕННО НЕ ВКЛЮЧЕНО:
|
||
# .env (режим 600) и runner*/data/ с регистрационными токенами — секреты.
|
||
# Они остаются ручным off-box пунктом владельца, тем же, что .env.runtime
|
||
# в приёмке #2203. Раннеры при восстановлении перерегистрируются из UI за
|
||
# пару минут; компоуз-файл руками не восстановишь.
|
||
config_out="${LOCAL_BACKUP_DIR}/forgejo-config_${ts}.tar.gz"
|
||
log "Bundling Forgejo compose config from ${FORGEJO_DIR} -> ${config_out}"
|
||
|
||
# -C + явные имена, а не путь целиком: в архиве оказываются только те файлы,
|
||
# что перечислены, без каталога data/ (он огромен и уже покрыт repo-бандлом).
|
||
if ! tar -czf "$config_out" -C "$FORGEJO_DIR" \
|
||
--exclude='.env' \
|
||
$(cd "$FORGEJO_DIR" && ls *.yml 2>/dev/null) 2>/dev/null; then
|
||
log "WARN: не удалось собрать config-бандл — продолжаю без него" >&2
|
||
rm -f "$config_out"
|
||
config_out=""
|
||
fi
|
||
|
||
if [[ -n "$config_out" ]]; then
|
||
# Порог намеренно низкий: файлы и правда маленькие. Проверяем не «объём», а
|
||
# «архив собрался и читается» — те же две проверки, что у repo-бандла.
|
||
config_bytes=$(wc -c < "$config_out" | tr -d ' ')
|
||
if (( config_bytes < 200 )) || ! tar -tzf "$config_out" >/dev/null 2>&1; then
|
||
log "WARN: config-бандл подозрителен (${config_bytes} байт / не читается) — не выгружаю" >&2
|
||
rm -f "$config_out"
|
||
config_out=""
|
||
else
|
||
log "Config bundle OK: ${config_out} (${config_bytes} байт, файлов: $(tar -tzf "$config_out" | wc -l))"
|
||
fi
|
||
fi
|
||
|
||
# --- 4. upload to S3 (mandatory — see guard 0 above) ---
|
||
for f in "$db_out" "$repos_out" ${config_out:+"$config_out"}; do
|
||
key="${FORGEJO_S3_PREFIX%/}/$(basename "$f")"
|
||
log "Uploading to s3://${FORGEJO_S3_BUCKET}/${key}"
|
||
# Same AWS_CA_BUNDLE fix as ops/backup.sh / backup-tradein-db.sh — aws-cli
|
||
# v2's own baked-in CA bundle is missing the root Selectel's cert chains
|
||
# to; point it at the container's system store instead.
|
||
docker run --rm \
|
||
-e AWS_ACCESS_KEY_ID="$FORGEJO_S3_ACCESS_KEY" \
|
||
-e AWS_SECRET_ACCESS_KEY="$FORGEJO_S3_SECRET_KEY" \
|
||
-e AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
|
||
-v "$LOCAL_BACKUP_DIR":/backup:ro \
|
||
amazon/aws-cli:latest \
|
||
--endpoint-url "$FORGEJO_S3_ENDPOINT" \
|
||
s3 cp --no-progress "/backup/$(basename "$f")" "s3://${FORGEJO_S3_BUCKET}/${key}"
|
||
done
|
||
log "S3 upload OK"
|
||
|
||
# --- 4. local retention ---
|
||
# shellcheck disable=SC2012
|
||
ls -1t "$LOCAL_BACKUP_DIR"/forgejo-db_*.sql.gz 2>/dev/null \
|
||
| tail -n +"$((KEEP + 1))" \
|
||
| xargs -r rm -f
|
||
# shellcheck disable=SC2012
|
||
ls -1t "$LOCAL_BACKUP_DIR"/forgejo-repos_*.tar.gz 2>/dev/null \
|
||
| tail -n +"$((KEEP + 1))" \
|
||
| xargs -r rm -f
|
||
|
||
shopt -s nullglob
|
||
remaining_db=( "$LOCAL_BACKUP_DIR"/forgejo-db_*.sql.gz )
|
||
remaining_repos=( "$LOCAL_BACKUP_DIR"/forgejo-repos_*.tar.gz )
|
||
shopt -u nullglob
|
||
log "Forgejo backup done. Local copies retained: ${#remaining_db[@]} db + ${#remaining_repos[@]} repo bundles (KEEP=${KEEP})."
|
||
|
||
# --- missed-run detection (#2203): mark success ONLY after everything above
|
||
# (including the mandatory S3 upload) has passed. ---
|
||
write_sentinel "$SENTINEL_FILE"
|