gendesign/ops/backup-forgejo.sh

#!/usr/bin/env bash
# Backup for self-hosted Forgejo (git.gendsgn.ru) — repos + DB, off-box to S3
# under its OWN, more-restricted key (#2203).
#
# Forgejo is deployed OUTSIDE this repo's checkout, per
# infra/Forgejo_Migration_BotServer_To_Beget_2026-05-16.md (vault):
#   - compose dir on the VM: /home/gendesign/forgejo/
#   - repo data:             /home/gendesign/forgejo/data/forgejo/git/repositories/
#   - DB: NOT a dedicated container — a separate `forgejo` user+database
#     inside the SAME shared postgres container as the main app
#     (gendesign-postgres-1). `pg_dump -U forgejo forgejo` via `docker exec`.
#
# Two artifacts per run:
#   1. DB dump   — `pg_dump --no-owner --clean --if-exists` (repos/issues/PRs/
#      users/settings/Actions config — everything except the git data itself).
#   2. Repo bundle — a tar.gz of the bare-repo tree under FORGEJO_REPOS_DIR.
#      This is functionally a "bundle of repositories": bare repos already
#      ARE just refs+objects, byte-identical to what `git bundle` would
#      capture, and tar-ing the whole tree in one shot is far more robust
#      than enumerating individual repos (new repos need zero script changes,
#      nothing to keep in sync). LFS objects/attachments/avatars under
#      FORGEJO_DATA_DIR are OUT of scope for this pass — repo code + full DB
#      (which has LFS pointers, not the blobs) covers the "backup the code"
#      ask; note this gap in the PR if larger asset backup is wanted later.
#
# Off-box S3 upload is MANDATORY here, not optional like ops/backup.sh —
# the whole point of this script is getting Forgejo's code off the VM it
# lives on, so with no S3 creds configured this refuses to run rather than
# silently producing a local-only "backup" that doesn't meet that bar.
#
# The S3 key here is a SEPARATE, narrower-scoped service user than the one
# ops/backup.sh and backup-tradein-db.sh use (`gendsgn-backup-writer`, which
# can write anywhere in the bucket root) — this one can ONLY PutObject under
# s3://gendsgn-backups/forgejo/*, nothing else, so a compromised Forgejo host
# can't touch the main/tradein DB backups sitting in the same bucket. See the
# PR description for the exact bucket policy JSON (mirrors the existing
# gendsgn-backup-writer policy pattern — vault meta/00_credentials.md
# "Selectel S3 — бэкап-бакет gendsgn-backups").
#
# THE KEY DOES NOT EXIST YET (as of #2203) — a human needs to create the
# service user + policy in the Selectel panel first. Until then this script
# is expected to fail loudly and exit non-zero; that is the correct,
# intentional behaviour, not a bug.
#
# Missed-run detection (#2203): same sentinel/staleness-check pattern as
# ops/backup.sh — see ops/check-backup-staleness.sh for the alerting side.
#
# Usage (cron — `bash <path>`, not a bare path, same "+x can go missing on a
# raw deploy reset" reasoning as the other two backup scripts, #71):
#   15 4 * * * bash /opt/gendesign/ops/backup-forgejo.sh >> /var/log/gendesign-backup-forgejo.log 2>&1
#
# Config file (root-owned, chmod 600, NOT in git) — see
# ops/gendesign-backup-forgejo.default.example for the full list:
#   FORGEJO_S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru
#   FORGEJO_S3_BUCKET=gendsgn-backups
#   FORGEJO_S3_PREFIX=forgejo/
#   FORGEJO_S3_ACCESS_KEY=...
#   FORGEJO_S3_SECRET_KEY=...

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=./lib-backup.sh
source "$SCRIPT_DIR/lib-backup.sh"

# --- config (env-overridable) ---
FORGEJO_DIR="${FORGEJO_DIR:-/home/gendesign/forgejo}"
FORGEJO_DATA_DIR="${FORGEJO_DATA_DIR:-${FORGEJO_DIR}/data/forgejo}"
FORGEJO_REPOS_DIR="${FORGEJO_REPOS_DIR:-${FORGEJO_DATA_DIR}/git/repositories}"

PG_CONTAINER="${PG_CONTAINER:-gendesign-postgres-1}"
FORGEJO_DB_USER="${FORGEJO_DB_USER:-forgejo}"
FORGEJO_DB_NAME="${FORGEJO_DB_NAME:-forgejo}"

LOCAL_BACKUP_DIR="${LOCAL_BACKUP_DIR:-/opt/gendesign/backups/forgejo}"
KEEP="${KEEP:-7}"
MIN_DB_DUMP_BYTES="${MIN_DB_DUMP_BYTES:-2048}"        # forgejo DB is small (config/issues/PRs, no git blobs)
MIN_REPOS_BUNDLE_BYTES="${MIN_REPOS_BUNDLE_BYTES:-10240}"
SENTINEL_FILE="${SENTINEL_FILE:-${LOCAL_BACKUP_DIR}/.last_success}"

FORGEJO_BACKUP_ENV_FILE="${FORGEJO_BACKUP_ENV_FILE:-/etc/default/gendesign-backup-forgejo}"
# shellcheck source=/dev/null
[[ -f "$FORGEJO_BACKUP_ENV_FILE" ]] && source "$FORGEJO_BACKUP_ENV_FILE"

FORGEJO_S3_ENDPOINT="${FORGEJO_S3_ENDPOINT:-}"
FORGEJO_S3_BUCKET="${FORGEJO_S3_BUCKET:-}"
FORGEJO_S3_PREFIX="${FORGEJO_S3_PREFIX:-forgejo/}"
FORGEJO_S3_ACCESS_KEY="${FORGEJO_S3_ACCESS_KEY:-}"
FORGEJO_S3_SECRET_KEY="${FORGEJO_S3_SECRET_KEY:-}"

# --- guard 0: S3 creds MUST be configured — this backup's whole point is
#     off-box storage under a narrow-scoped key. No creds = refuse to run,
#     loudly, rather than silently producing a local-only "backup" (unlike
#     ops/backup.sh, where local-only is an accepted fallback). ---
if [[ -z "$FORGEJO_S3_ENDPOINT" || -z "$FORGEJO_S3_BUCKET" || -z "$FORGEJO_S3_ACCESS_KEY" || -z "$FORGEJO_S3_SECRET_KEY" ]]; then
    log "ERROR: Forgejo S3 backup is NOT CONFIGURED."
    log "ERROR: missing one or more of FORGEJO_S3_ENDPOINT / FORGEJO_S3_BUCKET / FORGEJO_S3_ACCESS_KEY / FORGEJO_S3_SECRET_KEY."
    log "ERROR: create the narrow-scoped 'gendsgn-backup-forgejo-writer' S3 service user (see PR for the exact bucket policy)"
    log "ERROR: and fill them into ${FORGEJO_BACKUP_ENV_FILE} (see ops/gendesign-backup-forgejo.default.example)."
    log "ERROR: refusing to produce a local-only backup — that does not meet this script's off-box requirement."
    exit 1
fi

if [[ ! -d "$FORGEJO_REPOS_DIR" ]]; then
    log "ERROR: FORGEJO_REPOS_DIR does not exist -> $FORGEJO_REPOS_DIR"
    log "ERROR: Forgejo's on-disk layout may differ from the documented default — override FORGEJO_REPOS_DIR."
    exit 1
fi

mkdir -p "$LOCAL_BACKUP_DIR"
ts=$(date -u +'%Y%m%d_%H%M%S')
db_out="${LOCAL_BACKUP_DIR}/forgejo-db_${ts}.sql.gz"
repos_out="${LOCAL_BACKUP_DIR}/forgejo-repos_${ts}.tar.gz"

# --- 1. DB dump ---
log "Dumping Forgejo DB (${FORGEJO_DB_NAME} as ${FORGEJO_DB_USER}, container ${PG_CONTAINER}) -> ${db_out}"
docker exec "$PG_CONTAINER" pg_dump -U "$FORGEJO_DB_USER" -d "$FORGEJO_DB_NAME" --no-owner --clean --if-exists \
    | gzip -9 > "$db_out"

if [[ ! -s "$db_out" ]]; then
    log "ERROR: DB dump is empty -> $db_out — removing, keeping previous good backups." >&2
    rm -f "$db_out"
    exit 1
fi

db_bytes=$(wc -c < "$db_out" | tr -d ' ')
if (( db_bytes < MIN_DB_DUMP_BYTES )); then
    log "ERROR: DB dump only ${db_bytes} bytes (< ${MIN_DB_DUMP_BYTES} floor) — likely a failed dump." >&2
    rm -f "$db_out"
    exit 1
fi

if ! verify_dump_integrity "$db_out" "-- PostgreSQL database dump complete" "forgejo DB dump"; then
    log "Removing suspect DB dump, NOT pruning older good backups." >&2
    rm -f "$db_out"
    exit 1
fi

log "DB dump OK: ${db_out} ($(du -h "$db_out" | cut -f1), ${db_bytes} bytes)"

# --- 2. repo bundle (tar.gz of the bare-repo tree) ---
log "Bundling Forgejo repos from ${FORGEJO_REPOS_DIR} -> ${repos_out}"
tar -czf "$repos_out" -C "$(dirname "$FORGEJO_REPOS_DIR")" "$(basename "$FORGEJO_REPOS_DIR")"

if [[ ! -s "$repos_out" ]]; then
    log "ERROR: repo bundle is empty -> $repos_out — removing." >&2
    rm -f "$repos_out"
    exit 1
fi

repos_bytes=$(wc -c < "$repos_out" | tr -d ' ')
if (( repos_bytes < MIN_REPOS_BUNDLE_BYTES )); then
    log "ERROR: repo bundle only ${repos_bytes} bytes (< ${MIN_REPOS_BUNDLE_BYTES} floor) — likely empty/failed FORGEJO_REPOS_DIR." >&2
    rm -f "$repos_out"
    exit 1
fi

# tar integrity: a bundle truncated mid-write (disk full, killed process)
# fails `tar -t` even when the gzip framing looks superficially OK — same
# "don't trust non-empty + big-enough alone" discipline as verify_dump_integrity.
if ! tar -tzf "$repos_out" >/dev/null 2>&1; then
    log "ERROR: repo bundle failed tar integrity check -> $repos_out (truncated archive?)" >&2
    rm -f "$repos_out"
    exit 1
fi

log "Repo bundle OK: ${repos_out} ($(du -h "$repos_out" | cut -f1), ${repos_bytes} bytes)"

# --- 3. upload to S3 (mandatory — see guard 0 above) ---
for f in "$db_out" "$repos_out"; do
    key="${FORGEJO_S3_PREFIX%/}/$(basename "$f")"
    log "Uploading to s3://${FORGEJO_S3_BUCKET}/${key}"
    # Same AWS_CA_BUNDLE fix as ops/backup.sh / backup-tradein-db.sh — aws-cli
    # v2's own baked-in CA bundle is missing the root Selectel's cert chains
    # to; point it at the container's system store instead.
    docker run --rm \
        -e AWS_ACCESS_KEY_ID="$FORGEJO_S3_ACCESS_KEY" \
        -e AWS_SECRET_ACCESS_KEY="$FORGEJO_S3_SECRET_KEY" \
        -e AWS_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt \
        -v "$LOCAL_BACKUP_DIR":/backup:ro \
        amazon/aws-cli:latest \
        --endpoint-url "$FORGEJO_S3_ENDPOINT" \
        s3 cp --no-progress "/backup/$(basename "$f")" "s3://${FORGEJO_S3_BUCKET}/${key}"
done
log "S3 upload OK"

# --- 4. local retention ---
# shellcheck disable=SC2012
ls -1t "$LOCAL_BACKUP_DIR"/forgejo-db_*.sql.gz 2>/dev/null \
    | tail -n +"$((KEEP + 1))" \
    | xargs -r rm -f
# shellcheck disable=SC2012
ls -1t "$LOCAL_BACKUP_DIR"/forgejo-repos_*.tar.gz 2>/dev/null \
    | tail -n +"$((KEEP + 1))" \
    | xargs -r rm -f

shopt -s nullglob
remaining_db=( "$LOCAL_BACKUP_DIR"/forgejo-db_*.sql.gz )
remaining_repos=( "$LOCAL_BACKUP_DIR"/forgejo-repos_*.tar.gz )
shopt -u nullglob
log "Forgejo backup done. Local copies retained: ${#remaining_db[@]} db + ${#remaining_repos[@]} repo bundles (KEEP=${KEEP})."

# --- missed-run detection (#2203): mark success ONLY after everything above
#     (including the mandatory S3 upload) has passed. ---
write_sentinel "$SENTINEL_FILE"