From 6883d1417748ecd3eed421e53e69e4ed4fa71ac6 Mon Sep 17 00:00:00 2001 From: Light1YT Date: Sat, 13 Jun 2026 20:13:04 +0500 Subject: [PATCH] fix(ops): repair broken main-DB backup + harden auth scripts/docs (#71 #427 #429 #428) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #71 (CRITICAL): backup.sh committed 100644 → git reset --hard on deploy re-asserts non-exec mode → raw-path cron fails Permission denied (last good dump 2026-05-27, no S3). Commit 100755 + chmod ops/*.sh in deploy.yml + size sanity-check (never prune good dumps for a truncated one) + keep-N retention + optional S3 (redacted /etc/default template). Modeled on the working backup-tradein-db.sh. #427: widen basic_auth username regex ^[a-z][a-z0-9_.-]{1,62}$ + literal-escape dotted names in grep probes. #429: replace list_users.sh false-positive grep with anchored awk over basic_auth block. #428: PILOT_ACCESS.md support email → pilot@gendsgn.ru. Closes #71 Closes #427 Closes #429 Closes #428 --- .forgejo/workflows/deploy.yml | 7 ++ docs/PILOT_ACCESS.md | 2 +- ops/backup.sh | 119 +++++++++++++++++++-------- ops/gendesign-backup.default.example | 21 +++++ scripts/auth/add_user.sh | 14 +++- scripts/auth/list_users.sh | 54 +++++++++--- scripts/auth/remove_user.sh | 14 ++-- 7 files changed, 176 insertions(+), 55 deletions(-) mode change 100644 => 100755 ops/backup.sh create mode 100644 ops/gendesign-backup.default.example diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index f3529b40..be9b0ffb 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -192,6 +192,13 @@ jobs: git fetch origin main git reset --hard origin/main + # Re-assert +x on ops scripts (#71). These are committed 100755, so a + # reset normally preserves the bit — but this is belt-and-suspenders so + # a script that ever lands as 644 can't silently break its cron caller + # (cron `30 3 * * * bash /opt/gendesign/ops/backup.sh` is +x-independent, + # but other callers may invoke the raw path). + chmod +x ops/*.sh 2>/dev/null || true + # Sentry release tracking mkdir -p backend touch backend/.env.runtime diff --git a/docs/PILOT_ACCESS.md b/docs/PILOT_ACCESS.md index ef8e9b1c..ce923850 100644 --- a/docs/PILOT_ACCESS.md +++ b/docs/PILOT_ACCESS.md @@ -7,5 +7,5 @@ При первом заходе браузер запросит данные — введите выше. Credentials кэшируются до закрытия окна браузера. -Если забыли пароль — напишите claudestars@proton.me, мы выпустим новый. +Если забыли пароль — напишите pilot@gendsgn.ru, мы выпустим новый. По завершении пилота доступ будет отозван. diff --git a/ops/backup.sh b/ops/backup.sh old mode 100644 new mode 100755 index 337ad755..348518f7 --- a/ops/backup.sh +++ b/ops/backup.sh @@ -1,73 +1,122 @@ #!/usr/bin/env bash -# Daily Postgres backup script. Designed to run from cron on the prod VM. +# Daily Postgres backup for the MAIN gendesign DB. Runs from cron on the prod VM. # -# Usage (one-time setup on VM): -# sudo cp /opt/gendesign/ops/backup.sh /usr/local/bin/gendesign-backup -# sudo chmod +x /usr/local/bin/gendesign-backup -# crontab -e -# # Run daily at 03:30 Moscow time -# 30 3 * * * /usr/local/bin/gendesign-backup >> /var/log/gendesign-backup.log 2>&1 +# Modeled on the proven ops/.../backup-tradein-db.sh (#397). Hardened for #71 +# after the main-DB backup silently broke (last good dump 2026-05-27): a raw +# `git reset --hard origin/main` on every deploy reset this file's mode to 644, +# so a cron entry that invoked the raw path got "Permission denied" every run. # -# Optional S3 upload — set these env vars in /etc/default/gendesign-backup -# (or as `Environment=` in a systemd timer). Without them, dumps stay local only. +# Robustness measures here: +# - cron should invoke via `bash ` so a missing +x bit can't break it +# (this file is ALSO committed 100755, and deploy.yml re-chmods ops/*.sh); +# - sanity-check: a suspiciously small dump (< MIN_DUMP_BYTES) is treated as a +# failed dump — it's deleted and the script exits non-zero, so a good prior +# dump is never pruned in favour of a truncated one; +# - retention: keep the KEEP most-recent local dumps, delete the rest. # +# Usage (cron — note `bash`, not a bare path, so +x is irrelevant): +# 30 3 * * * bash /opt/gendesign/ops/backup.sh >> /var/log/gendesign-backup.log 2>&1 +# +# Optional S3 upload — set these in /etc/default/gendesign-backup (root-owned, +# chmod 600, NOT in git). Without them, dumps stay local only: # S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru # Selectel S3 # S3_BUCKET=gendesign-backups # S3_ACCESS_KEY=... # S3_SECRET_KEY=... +# A redacted template lives at ops/gendesign-backup.default.example. # -# Retention: deletes local dumps older than 7 days. S3 retention policy should -# be configured at the bucket level (lifecycle rule), not in this script. +# Restore: see ops/restore.sh. set -euo pipefail -# --- config --- -COMPOSE_DIR=/opt/gendesign -COMPOSE_FILE=docker-compose.prod.yml -LOCAL_BACKUP_DIR=/opt/gendesign/backups -RETENTION_DAYS=7 +# --- config (env-overridable) --- +COMPOSE_DIR="${COMPOSE_DIR:-/opt/gendesign}" +COMPOSE_FILE="${COMPOSE_FILE:-docker-compose.prod.yml}" +COMPOSE_PROJECT="${COMPOSE_PROJECT:-gendesign}" +LOCAL_BACKUP_DIR="${LOCAL_BACKUP_DIR:-/opt/gendesign/backups}" +KEEP="${KEEP:-7}" # how many recent local dumps to keep +MIN_DUMP_BYTES="${MIN_DUMP_BYTES:-51200}" # 50 KiB floor; gzip'd schema-only dump + # is already > this, so a healthy dump + # never trips it. Real DB is far larger. # Optional S3 env. Loaded from /etc/default/gendesign-backup if present. [[ -f /etc/default/gendesign-backup ]] && source /etc/default/gendesign-backup +log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*"; } + # --- run --- mkdir -p "$LOCAL_BACKUP_DIR" -TIMESTAMP=$(date +%Y%m%d_%H%M%S) -DUMP_FILE="${LOCAL_BACKUP_DIR}/gendesign_${TIMESTAMP}.sql.gz" +# Underscore naming kept (gendesign_YYYYMMDD_HHMMSS.sql.gz) so this matches +# pre-existing prod dumps and restore.sh's example path. UTC for stable ordering. +ts=$(date -u +'%Y%m%d_%H%M%S') +out="${LOCAL_BACKUP_DIR}/gendesign_${ts}.sql.gz" cd "$COMPOSE_DIR" -# Read DB user/name from running compose env. Falls back to "gendesign" if env empty. -DB_USER=$(docker compose -f "$COMPOSE_FILE" exec -T postgres printenv POSTGRES_USER 2>/dev/null | tr -d '\r' || true) -DB_NAME=$(docker compose -f "$COMPOSE_FILE" exec -T postgres printenv POSTGRES_DB 2>/dev/null | tr -d '\r' || true) +compose() { docker compose -p "$COMPOSE_PROJECT" -f "$COMPOSE_FILE" "$@"; } + +# Read DB user/name from the running compose env; fall back to "gendesign". +DB_USER=$(compose exec -T postgres printenv POSTGRES_USER 2>/dev/null | tr -d '\r' || true) +DB_NAME=$(compose exec -T postgres printenv POSTGRES_DB 2>/dev/null | tr -d '\r' || true) DB_USER=${DB_USER:-gendesign} DB_NAME=${DB_NAME:-gendesign} -echo "[$(date -Is)] Dumping ${DB_NAME} as ${DB_USER} → ${DUMP_FILE}" +log "Dumping ${DB_NAME} as ${DB_USER} -> ${out}" -docker compose -f "$COMPOSE_FILE" exec -T postgres \ - pg_dump --clean --if-exists -U "$DB_USER" "$DB_NAME" \ - | gzip -9 > "$DUMP_FILE" +# --clean --if-exists -> dump is self-sufficient for restore from scratch. +# --no-owner -> restore does not require identical roles. +# `set -o pipefail` (from set -euo pipefail) makes a pg_dump failure fail the +# whole pipe, so a Postgres error can't yield a "successful" tiny gzip. +compose exec -T postgres \ + pg_dump -U "$DB_USER" -d "$DB_NAME" --no-owner --clean --if-exists 2>/dev/null \ + | gzip -9 > "$out" -DUMP_SIZE=$(stat -c '%s' "$DUMP_FILE") -echo "[$(date -Is)] Dump complete: ${DUMP_SIZE} bytes" +# --- sanity-check: refuse to keep (and thus never prune good dumps for) a +# truncated/empty dump. --- +if [[ ! -s "$out" ]]; then + log "ERROR: dump is empty -> $out — removing, keeping previous good dumps." >&2 + rm -f "$out" + exit 1 +fi -# --- optional S3 upload --- +dump_bytes=$(wc -c < "$out" | tr -d ' ') +if (( dump_bytes < MIN_DUMP_BYTES )); then + log "ERROR: dump only ${dump_bytes} bytes (< ${MIN_DUMP_BYTES} floor) — likely a failed dump." >&2 + log "Removing suspect dump, NOT pruning older good dumps." >&2 + rm -f "$out" + exit 1 +fi + +log "Dump OK: ${out} ($(du -h "$out" | cut -f1), ${dump_bytes} bytes)" + +# --- optional S3 upload (only if all four vars present) --- if [[ -n "${S3_ENDPOINT:-}" && -n "${S3_BUCKET:-}" && -n "${S3_ACCESS_KEY:-}" && -n "${S3_SECRET_KEY:-}" ]]; then - echo "[$(date -Is)] Uploading to s3://${S3_BUCKET}/" + log "Uploading to s3://${S3_BUCKET}/$(basename "$out")" docker run --rm \ -e AWS_ACCESS_KEY_ID="$S3_ACCESS_KEY" \ -e AWS_SECRET_ACCESS_KEY="$S3_SECRET_KEY" \ -v "$LOCAL_BACKUP_DIR":/backup:ro \ amazon/aws-cli:latest \ --endpoint-url "$S3_ENDPOINT" \ - s3 cp "/backup/$(basename "$DUMP_FILE")" "s3://${S3_BUCKET}/" - echo "[$(date -Is)] S3 upload OK" + s3 cp "/backup/$(basename "$out")" "s3://${S3_BUCKET}/" + log "S3 upload OK" else - echo "[$(date -Is)] S3 vars not set — backup stays local only" + log "S3 vars not set — backup stays local only" fi -# --- prune old local dumps --- -find "$LOCAL_BACKUP_DIR" -name 'gendesign_*.sql.gz' -mtime "+${RETENTION_DAYS}" -print -delete +# --- retention: keep the KEEP most-recent local dumps, delete the rest. +# Runs only AFTER a verified-good dump above, so a failed run (which exits +# early) can never delete older good dumps. --- +# SC2012: ls is fine here — filenames are fully controlled (gendesign_.sql.gz, +# no spaces/newlines) and we need ls's -t mtime sort for "keep newest N" (same +# pattern as the proven backup-tradein-db.sh). +# shellcheck disable=SC2012 +ls -1t "$LOCAL_BACKUP_DIR"/gendesign_*.sql.gz 2>/dev/null \ + | tail -n +"$((KEEP + 1))" \ + | xargs -r rm -f -echo "[$(date -Is)] Backup done." +# Count via a glob array (no ls parsing). nullglob -> empty array if no match. +shopt -s nullglob +remaining=( "$LOCAL_BACKUP_DIR"/gendesign_*.sql.gz ) +shopt -u nullglob +log "Backup done. Local dumps retained: ${#remaining[@]} (KEEP=${KEEP})." diff --git a/ops/gendesign-backup.default.example b/ops/gendesign-backup.default.example new file mode 100644 index 00000000..dea81c3c --- /dev/null +++ b/ops/gendesign-backup.default.example @@ -0,0 +1,21 @@ +# Environment file for ops/backup.sh (MAIN gendesign DB backup, #71). +# +# Install on the prod VM as a ROOT-OWNED, chmod-600 file that is NOT in git: +# sudo cp /opt/gendesign/ops/gendesign-backup.default.example /etc/default/gendesign-backup +# sudo chmod 600 /etc/default/gendesign-backup +# sudo $EDITOR /etc/default/gendesign-backup # fill in real S3 credentials +# +# backup.sh sources this file if present. With NO S3 vars set, dumps stay +# local-only under /opt/gendesign/backups (retention KEEP=7). Fill these in to +# also push each dump off-box to S3 (recommended — local-only dies with the VM). + +# --- S3 off-site upload (Selectel S3-compatible). All four required to enable. --- +#S3_ENDPOINT=https://s3.ru-1.storage.selcloud.ru +#S3_BUCKET=gendesign-backups +#S3_ACCESS_KEY=REPLACE_WITH_REAL_ACCESS_KEY +#S3_SECRET_KEY=REPLACE_WITH_REAL_SECRET_KEY + +# --- optional overrides (defaults are sensible; uncomment only to change) --- +#KEEP=7 # how many recent local dumps to retain +#MIN_DUMP_BYTES=51200 # sanity floor; a dump smaller than this is treated as failed +#LOCAL_BACKUP_DIR=/opt/gendesign/backups diff --git a/scripts/auth/add_user.sh b/scripts/auth/add_user.sh index 6232595f..b3893f23 100644 --- a/scripts/auth/add_user.sh +++ b/scripts/auth/add_user.sh @@ -28,15 +28,21 @@ USERNAME="$1" COMMENT="$2" DATE="$(date -u +%Y-%m-%d)" -# Validate username: ^[a-z][a-z0-9-]{2,30}$ -if ! [[ "$USERNAME" =~ ^[a-z][a-z0-9-]{2,30}$ ]]; then - echo "ERROR: invalid username '$USERNAME'. Must match ^[a-z][a-z0-9-]{2,30}$" >&2 +# Validate username: ^[a-z][a-z0-9_.-]{1,62}$ +# Allows dot/underscore (john.doe, jane_smith) and 2-char names; starts with a +# letter. 63-char cap keeps it sane for a Caddyfile basic_auth entry (#427). +if ! [[ "$USERNAME" =~ ^[a-z][a-z0-9_.-]{1,62}$ ]]; then + echo "ERROR: invalid username '$USERNAME'. Must match ^[a-z][a-z0-9_.-]{1,62}$" >&2 exit 1 fi # ─── idempotency guard ──────────────────────────────────────────────────────── -if grep -qP "^\s+${USERNAME}\s" "$SNIPPET" 2>/dev/null; then +# Escape regex metachars (notably the now-allowed '.') so the username is matched +# literally in the existence probe below (#427). +USERNAME_RE=$(printf '%s' "$USERNAME" | sed -E 's/[.[\^$*+?(){}|]/\\&/g') + +if grep -qP "^\s+${USERNAME_RE}\s" "$SNIPPET" 2>/dev/null; then echo "ERROR: user '$USERNAME' already exists in $SNIPPET. Remove first with remove_user.sh." >&2 exit 1 fi diff --git a/scripts/auth/list_users.sh b/scripts/auth/list_users.sh index e8f7a024..e151bde3 100644 --- a/scripts/auth/list_users.sh +++ b/scripts/auth/list_users.sh @@ -18,14 +18,48 @@ echo "" printf "%-12s %s\n" "USERNAME" "COMMENT" printf "%-12s %s\n" "--------" "-------" -# Extract lines that look like user entries (indented username + base64 hash + optional comment) -# Format: username # comment -grep -P '^\s+[a-z][a-z0-9-]+\s+[A-Za-z0-9+/]+=*' "$SNIPPET" | \ - sed -E 's/^\s+([a-z][a-z0-9-]+)\s+\S+\s*(#\s*)?(.*)$/\1\t\3/' | \ - while IFS=$'\t' read -r user comment; do - printf "%-12s %s\n" "$user" "$comment" - done +# Parse only the real user lines INSIDE the `basic_auth ... { ... }` block (#429). +# The previous grep `^\s+[a-z][a-z0-9-]+\s+[A-Za-z0-9+/]+=*` also matched indented +# base64-ish comment text and miscounted. Here we: +# - enter the block on a line whose first token is `basic_auth` ending in `{`; +# - leave it on the closing `}`; +# - inside, skip blank and comment-only lines, require a username-shaped first +# field (^[a-z][a-z0-9_.-]+$, matches add/remove_user.sh) AND a 2nd field +# (the hash). Comment is whatever follows the first `#`. +# A single pass prints rows and the total, so the count can't drift from the list. +awk ' + $1 == "basic_auth" && /\{[[:space:]]*$/ { in_block = 1; next } + in_block && /^[[:space:]]*\}/ { in_block = 0; next } + !in_block { next } -echo "" -TOTAL=$(grep -cP '^\s+[a-z][a-z0-9-]+\s+[A-Za-z0-9+/]+=*' "$SNIPPET" || true) -echo "Total: $TOTAL user(s)" + { + line = $0 + sub(/^[[:space:]]+/, "", line) # strip leading indent + if (line == "" || line ~ /^#/) next # blank / comment-only + + user = line + sub(/[[:space:]].*$/, "", user) # first whitespace-delimited token + if (user !~ /^[a-z][a-z0-9_.-]+$/) next # must look like a username + + rest = line + sub(/^[^[:space:]]+[[:space:]]+/, "", rest) # drop username + if (rest == "") next # no 2nd field -> not a user line + + hash = rest + sub(/[[:space:]].*$/, "", hash) # 2nd token = the hash + # Real entry: 2nd token is a base64-encoded bcrypt hash (base64 charset, + # long). This is what excludes indented prose like "note this ..." whose + # 2nd word ("this") is short non-base64 text — the #429 false positive. + if (hash !~ /^[A-Za-z0-9+\/=]{20,}$/) next + + comment = "" + h = index(rest, "#") + if (h > 0) { + comment = substr(rest, h + 1) + sub(/^[[:space:]]+/, "", comment) + } + printf "%-12s %s\n", user, comment + count++ + } + END { print ""; printf "Total: %d user(s)\n", count } +' "$SNIPPET" diff --git a/scripts/auth/remove_user.sh b/scripts/auth/remove_user.sh index 37796ee9..b2a4a38c 100644 --- a/scripts/auth/remove_user.sh +++ b/scripts/auth/remove_user.sh @@ -19,14 +19,18 @@ usage() { USERNAME="$1" -# Validate username format -if ! [[ "$USERNAME" =~ ^[a-z][a-z0-9-]{2,30}$ ]]; then - echo "ERROR: invalid username '$USERNAME'. Must match ^[a-z][a-z0-9-]{2,30}$" >&2 +# Validate username format: ^[a-z][a-z0-9_.-]{1,62}$ (matches add_user.sh, #427) +if ! [[ "$USERNAME" =~ ^[a-z][a-z0-9_.-]{1,62}$ ]]; then + echo "ERROR: invalid username '$USERNAME'. Must match ^[a-z][a-z0-9_.-]{1,62}$" >&2 exit 1 fi +# Escape regex metachars (notably the now-allowed '.') so the username is matched +# literally — otherwise `john.doe` could match/remove an unintended line (#427). +USERNAME_RE=$(printf '%s' "$USERNAME" | sed -E 's/[.[\^$*+?(){}|]/\\&/g') + # Check user exists -if ! grep -qP "^\s+${USERNAME}\s" "$SNIPPET" 2>/dev/null; then +if ! grep -qP "^\s+${USERNAME_RE}\s" "$SNIPPET" 2>/dev/null; then echo "ERROR: user '$USERNAME' not found in $SNIPPET" >&2 exit 1 fi @@ -35,7 +39,7 @@ fi TMP="$(mktemp)" trap 'rm -f "$TMP"' EXIT -grep -vP "^\s+${USERNAME}\s" "$SNIPPET" > "$TMP" +grep -vP "^\s+${USERNAME_RE}\s" "$SNIPPET" > "$TMP" mv "$TMP" "$SNIPPET" echo "Removed user '$USERNAME' from $SNIPPET"