From a79ee833f777000c1037bca8053482b150e13f60 Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 16 May 2026 13:38:04 +0300 Subject: [PATCH 1/2] feat(infra): add GlitchTip self-hosted error tracking (#204) Adds glitchtip-web + glitchtip-worker to docker-compose.prod.yml using the shared postgres/redis services. Caddy proxies errors.gendsgn.ru to glitchtip-web:8080 with auto-TLS. Bootstrap script handles DB creation and first-run setup idempotently. Env contract: GLITCHTIP_DB_PASS, GLITCHTIP_SECRET in .env; GLITCHTIP_DSN and NEXT_PUBLIC_GLITCHTIP_DSN populated after UI setup. --- .env.example | 7 +++ Caddyfile | 12 ++++ docker-compose.prod.yml | 40 ++++++++++++ scripts/bootstrap_glitchtip.sh | 108 +++++++++++++++++++++++++++++++++ 4 files changed, 167 insertions(+) create mode 100644 scripts/bootstrap_glitchtip.sh diff --git a/.env.example b/.env.example index fcfba003..1b16ba67 100644 --- a/.env.example +++ b/.env.example @@ -10,3 +10,10 @@ POSTGRES_PASSWORD=changeme # YC_REGISTRY_ID= # IMAGE_TAG=latest # NEXT_PUBLIC_API_BASE_URL=https://your-domain.example.ru + +# GlitchTip (errors.gendsgn.ru) +GLITCHTIP_DB_PASS= +GLITCHTIP_SECRET= +# DSN-ы из GlitchTip UI после первого старта (→ .env.runtime или backend/.env) +GLITCHTIP_DSN= +NEXT_PUBLIC_GLITCHTIP_DSN= diff --git a/Caddyfile b/Caddyfile index 5c52cc04..4b43b2ed 100644 --- a/Caddyfile +++ b/Caddyfile @@ -44,6 +44,18 @@ obsidian.gendsgn.ru { } } +# GlitchTip — self-hosted error tracking (Sentry-compatible). +# DNS: A-record errors.gendsgn.ru → IP VPS. +errors.gendsgn.ru { + encode zstd gzip + + reverse_proxy glitchtip-web:8080 + + log { + output file /var/log/caddy/errors.gendsgn.ru.log + } +} + # Plain HTTP by IP — kept for ssh-tunnel / debugging. # Caddy issues no TLS here (no hostname). :80 { diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 90632581..e48c9b99 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -121,6 +121,46 @@ services: condition: service_healthy command: ["celery", "-A", "app.workers.celery_app", "beat", "--loglevel=info"] + glitchtip-web: + image: glitchtip/glitchtip:latest + container_name: glitchtip-web + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_started + environment: + DATABASE_URL: postgres://glitchtip:${GLITCHTIP_DB_PASS}@postgres:5432/glitchtip + REDIS_URL: redis://redis:6379/2 + SECRET_KEY: ${GLITCHTIP_SECRET} + PORT: "8080" + EMAIL_URL: dummy:// + GLITCHTIP_DOMAIN: https://errors.gendsgn.ru + DEFAULT_FROM_EMAIL: errors@gendsgn.ru + ENABLE_USER_REGISTRATION: "false" + ENABLE_ORGANIZATION_CREATION: "false" + CELERY_WORKER_AUTOSCALE: "1,3" + restart: always + mem_limit: 256m + networks: [default] + + glitchtip-worker: + image: glitchtip/glitchtip:latest + container_name: glitchtip-worker + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_started + command: ./bin/run-celery-with-beat.sh + environment: + DATABASE_URL: postgres://glitchtip:${GLITCHTIP_DB_PASS}@postgres:5432/glitchtip + REDIS_URL: redis://redis:6379/2 + SECRET_KEY: ${GLITCHTIP_SECRET} + restart: always + mem_limit: 256m + networks: [default] + caddy: image: caddy:2 restart: unless-stopped diff --git a/scripts/bootstrap_glitchtip.sh b/scripts/bootstrap_glitchtip.sh new file mode 100644 index 00000000..18450a24 --- /dev/null +++ b/scripts/bootstrap_glitchtip.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# bootstrap_glitchtip.sh +# One-shot bootstrap for GlitchTip on the VPS. +# +# Run ONCE manually after merging feat/204-glitchtip-infra. +# Do NOT add to deploy.yml — this is idempotent setup, not a recurring step. +# +# Usage (on VPS via SSH): +# cd /opt/gendesign +# # Ensure GLITCHTIP_DB_PASS and GLITCHTIP_SECRET are set in .env +# bash scripts/bootstrap_glitchtip.sh +# +# After running: +# 1. docker compose -p gendesign -f docker-compose.prod.yml exec glitchtip-web ./manage.py createsuperuser +# 2. Log in to https://errors.gendsgn.ru → create org "gendesign" +# 3. Create projects: "backend", "frontend" +# 4. Copy DSNs → add to /opt/gendesign/backend/.env (GLITCHTIP_DSN) +# and /opt/gendesign/frontend/.env or .env.runtime (NEXT_PUBLIC_GLITCHTIP_DSN) +# 5. Recreate affected containers: +# docker compose -p gendesign -f docker-compose.prod.yml up -d --force-recreate --no-deps backend + +set -euo pipefail + +COMPOSE_FILE="docker-compose.prod.yml" +COMPOSE_PROJECT="gendesign" +COMPOSE="docker compose -p $COMPOSE_PROJECT -f $COMPOSE_FILE" + +# ── Load .env.runtime for GLITCHTIP_DB_PASS / POSTGRES_* ────────────────── +if [ -f ".env" ]; then + set -a; source .env; set +a +fi +if [ -f "backend/.env.runtime" ]; then + set -a; source backend/.env.runtime; set +a +fi + +: "${GLITCHTIP_DB_PASS:?GLITCHTIP_DB_PASS not set — add to .env or export}" +: "${POSTGRES_USER:?POSTGRES_USER not set}" +: "${POSTGRES_PASSWORD:?POSTGRES_PASSWORD not set}" +: "${POSTGRES_DB:?POSTGRES_DB not set}" + +echo "==> Ensuring postgres service is healthy..." +for i in $(seq 1 30); do + if $COMPOSE exec -T postgres pg_isready -U "$POSTGRES_USER" -q; then + break + fi + echo " Waiting for postgres... ($i/30)" + sleep 2 +done + +# ── Create glitchtip DB + user (idempotent) ──────────────────────────────── +echo "==> Creating glitchtip database and user (idempotent)..." +$COMPOSE exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ + -v ON_ERROR_STOP=on < Pulling glitchtip image..." +$COMPOSE pull glitchtip-web glitchtip-worker + +echo "==> Starting glitchtip-web and glitchtip-worker..." +$COMPOSE up -d glitchtip-web glitchtip-worker + +# ── Wait for glitchtip-web to become healthy ─────────────────────────────── +echo "==> Waiting for glitchtip-web to become healthy (up to 60s)..." +for i in $(seq 1 30); do + if $COMPOSE exec -T glitchtip-web curl -fsS http://localhost:8080/healthz/ >/dev/null 2>&1; then + echo " glitchtip-web is healthy." + break + fi + echo " Waiting... ($i/30)" + sleep 2 +done + +echo "" +echo "==> Bootstrap complete." +echo "" +echo "Next steps:" +echo " 1. Create superuser:" +echo " $COMPOSE exec glitchtip-web ./manage.py createsuperuser" +echo "" +echo " 2. Log in to https://errors.gendsgn.ru" +echo " - Create organization: gendesign" +echo " - Create project: backend (Platform: Python)" +echo " - Create project: frontend (Platform: JavaScript / Browser)" +echo "" +echo " 3. Copy DSNs from each project's Settings → DSN" +echo " - Add GLITCHTIP_DSN= to /opt/gendesign/backend/.env" +echo " - Add NEXT_PUBLIC_GLITCHTIP_DSN= to /opt/gendesign/backend/.env or .env" +echo "" +echo " 4. Recreate services to pick up new DSNs:" +echo " $COMPOSE up -d --force-recreate --no-deps backend" +echo " $COMPOSE up -d --force-recreate --no-deps worker beat" +echo "" +echo " 5. Verify: trigger a test exception and confirm it appears in GlitchTip." -- 2.45.3 From ffaa36336a0c0d0f676432aac81d98c692e9f77e Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sat, 16 May 2026 18:01:50 +0300 Subject: [PATCH 2/2] fix(infra): GlitchTip profiles + SQL safety + mem_limit (#204 review fixes) - Add profiles: ["glitchtip"] to glitchtip-web and glitchtip-worker so plain `compose up -d` does not start them before secrets/DB are ready - Fix SQL injection vector: pass password via psql -v variable + format(%L) instead of bare shell interpolation in heredoc - glitchtip-web mem_limit: 256m -> 512m (upstream minimum for Django+gunicorn) - glitchtip-worker mem_limit: 256m -> 384m (handles CELERY_WORKER_AUTOSCALE) - Move CELERY_WORKER_AUTOSCALE to worker only (no-op on gunicorn web service) - Pin image glitchtip/glitchtip:latest -> 6.1.6 - Add healthcheck to glitchtip-web (wget /api/0/) - bootstrap: use --profile glitchtip for pull/up; add COMPOSE_PROFILES note - bootstrap: use wget (consistent with compose healthcheck) in wait loop --- docker-compose.prod.yml | 21 ++++++++++++++++----- scripts/bootstrap_glitchtip.sh | 33 ++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 14 deletions(-) diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e48c9b99..e4d8f3a0 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -122,8 +122,12 @@ services: command: ["celery", "-A", "app.workers.celery_app", "beat", "--loglevel=info"] glitchtip-web: - image: glitchtip/glitchtip:latest + image: glitchtip/glitchtip:6.1.6 container_name: glitchtip-web + # profiles: ["glitchtip"] keeps this service from starting on plain `compose up -d`. + # Bootstrap script activates the profile after DB + secrets are ready. + # On subsequent deploys, set COMPOSE_PROFILES=glitchtip in /opt/gendesign/.env. + profiles: ["glitchtip"] depends_on: postgres: condition: service_healthy @@ -139,14 +143,20 @@ services: DEFAULT_FROM_EMAIL: errors@gendsgn.ru ENABLE_USER_REGISTRATION: "false" ENABLE_ORGANIZATION_CREATION: "false" - CELERY_WORKER_AUTOSCALE: "1,3" restart: always - mem_limit: 256m + mem_limit: 512m + healthcheck: + test: ["CMD", "wget", "-q", "--spider", "http://localhost:8080/api/0/"] + interval: 30s + timeout: 5s + retries: 5 + start_period: 60s networks: [default] glitchtip-worker: - image: glitchtip/glitchtip:latest + image: glitchtip/glitchtip:6.1.6 container_name: glitchtip-worker + profiles: ["glitchtip"] depends_on: postgres: condition: service_healthy @@ -157,8 +167,9 @@ services: DATABASE_URL: postgres://glitchtip:${GLITCHTIP_DB_PASS}@postgres:5432/glitchtip REDIS_URL: redis://redis:6379/2 SECRET_KEY: ${GLITCHTIP_SECRET} + CELERY_WORKER_AUTOSCALE: "1,3" restart: always - mem_limit: 256m + mem_limit: 384m networks: [default] caddy: diff --git a/scripts/bootstrap_glitchtip.sh b/scripts/bootstrap_glitchtip.sh index 18450a24..007bb5df 100644 --- a/scripts/bootstrap_glitchtip.sh +++ b/scripts/bootstrap_glitchtip.sh @@ -48,19 +48,24 @@ for i in $(seq 1 30); do done # ── Create glitchtip DB + user (idempotent) ──────────────────────────────── +# Password is passed via psql variable -v to avoid SQL injection from special +# chars (quotes, backslashes) in $GLITCHTIP_DB_PASS. format('%L', :'gt_pass') +# properly quotes the value server-side, so the shell never interpolates it +# into the SQL string. echo "==> Creating glitchtip database and user (idempotent)..." -$COMPOSE exec -T postgres psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" \ - -v ON_ERROR_STOP=on < Pulling glitchtip image..." -$COMPOSE pull glitchtip-web glitchtip-worker +$COMPOSE --profile glitchtip pull glitchtip-web glitchtip-worker echo "==> Starting glitchtip-web and glitchtip-worker..." -$COMPOSE up -d glitchtip-web glitchtip-worker +$COMPOSE --profile glitchtip up -d glitchtip-web glitchtip-worker + +echo "" +echo "NOTE: Add the following line to /opt/gendesign/.env so future deploys" +echo " keep GlitchTip running:" +echo " COMPOSE_PROFILES=glitchtip" # ── Wait for glitchtip-web to become healthy ─────────────────────────────── +# Use wget (available in glitchtip image) matching the compose healthcheck probe. echo "==> Waiting for glitchtip-web to become healthy (up to 60s)..." for i in $(seq 1 30); do - if $COMPOSE exec -T glitchtip-web curl -fsS http://localhost:8080/healthz/ >/dev/null 2>&1; then + if $COMPOSE exec -T glitchtip-web wget -q --spider http://localhost:8080/api/0/ >/dev/null 2>&1; then echo " glitchtip-web is healthy." break fi -- 2.45.3