From 58655b2971b14aec13e09b2bb875c540721199fc Mon Sep 17 00:00:00 2001 From: lekss361 Date: Sun, 24 May 2026 12:37:15 +0300 Subject: [PATCH] fix(deploy): resolve FDW bootstrap race + extract bootstrap DO block to .sql file MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three related fixes from PR #493 deploy incident (deploy/1156 failed at bootstrap; tradein-backend USER MAPPING never created): 1. deploy-tradein.yml — restart tradein-backend AFTER SQL migrations apply. Root cause: `compose up -d` runs at line 157 before migrations at line 163, so tradein-backend lifespan hook (ensure_fdw_user_mapping) tries to CREATE USER MAPPING against FOREIGN SERVER gendesign_remote that doesn't exist yet (created by 060_postgres_fdw_extension.sql later). Hook fails permanently with `psycopg.errors.UndefinedObject: server "gendesign_remote" does not exist`; never retried. Restart after migrations triggers retry. 2. deploy.yml — extract inline psql DO block into ops/db-bootstrap/ set_tradein_fdw_password.sql, run via `psql -v pw=$ENV -f `. Original block used multi-line backslash continuations inside double-quoted bash string with $$-dollar-quoting + psql var substitution — fragile escaping likely caused deploy/1156 failure between migration apply (09:01:03) and compose up -d. Clean .sql file with `-v pw=` substitution is safe and psql-native. 3. deploy.yml — verify postgres in gendesign_shared after `compose up -d`, force-recreate if not. Compose sometimes skips recreate on network membership change (PR #493 introduced `networks: shared (alias gendesign-postgres)` to postgres service). Without postgres in shared, tradein-postgres can't resolve DNS `gendesign-postgres` -> FDW queries fail with "could not translate host name". Manual fix on prod: `docker network connect --alias gendesign-postgres gendesign_shared gendesign-postgres-1`. No code changes — only deploy workflow and one new SQL file. --- .forgejo/workflows/deploy-tradein.yml | 11 +++++++ .forgejo/workflows/deploy.yml | 29 ++++++++++++------- ops/db-bootstrap/set_tradein_fdw_password.sql | 27 +++++++++++++++++ 3 files changed, 57 insertions(+), 10 deletions(-) create mode 100644 ops/db-bootstrap/set_tradein_fdw_password.sql diff --git a/.forgejo/workflows/deploy-tradein.yml b/.forgejo/workflows/deploy-tradein.yml index e27f3a2a..11ea64d6 100644 --- a/.forgejo/workflows/deploy-tradein.yml +++ b/.forgejo/workflows/deploy-tradein.yml @@ -169,6 +169,17 @@ jobs: || echo " (skipped — already applied or error: $fname)" done + # Retry backend lifespan hook AFTER migrations applied. + # tradein-backend startup runs ensure_fdw_user_mapping which needs + # FOREIGN SERVER gendesign_remote (created by 060_postgres_fdw_extension.sql). + # Without restart, the first compose-up's startup hook failed with + # "server gendesign_remote does not exist" because migrations hadn't run yet. + # See PR #493 deploy/1156 for the incident details. + echo "→ Restarting tradein-backend so lifespan hook retries USER MAPPING setup" + docker restart tradein-backend + # Give backend time to come up before Caddy reload + health check below + sleep 5 + # Caddy reload — основной Caddyfile содержит inline tradein routes # (см. Caddyfile в корне репы). Reload, чтобы Caddy перечитал DNS # tradein-backend / tradein-frontend (они в gendesign_shared network). diff --git a/.forgejo/workflows/deploy.yml b/.forgejo/workflows/deploy.yml index 12740547..f08b3336 100644 --- a/.forgejo/workflows/deploy.yml +++ b/.forgejo/workflows/deploy.yml @@ -252,22 +252,16 @@ jobs: echo "All migrations applied." # Set tradein_fdw_reader password from env (post-migration bootstrap). - # SQL migration creates role passwordless; password lives only in - # /opt/gendesign/backend/.env.runtime (sourced below via set -a / source). - # Idempotent: ALTER is no-op if password unchanged on subsequent deploys. + # SQL migration 100_tradein_fdw_role.sql creates role passwordless; + # password lives only in /opt/gendesign/backend/.env.runtime. + # See ops/db-bootstrap/set_tradein_fdw_password.sql for the idempotent DO block. set -a; source backend/.env.runtime; set +a if [ -n "${GENDESIGN_FDW_PASSWORD:-}" ]; then echo "→ Applying tradein_fdw_reader password from env" docker compose -p gendesign -f docker-compose.prod.yml exec -T postgres \ psql -U "$POSTGRES_USER" -d "$POSTGRES_DB" -v ON_ERROR_STOP=on \ -v "pw=$GENDESIGN_FDW_PASSWORD" \ - -c "DO \$\$ BEGIN \ - IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tradein_fdw_reader') THEN \ - EXECUTE format('ALTER ROLE tradein_fdw_reader WITH PASSWORD %L', :'pw'); \ - ELSE \ - RAISE NOTICE 'tradein_fdw_reader role missing; migration not yet applied'; \ - END IF; \ - END \$\$;" + < ops/db-bootstrap/set_tradein_fdw_password.sql else echo "⚠️ GENDESIGN_FDW_PASSWORD not set in backend/.env.runtime — skipping ALTER ROLE for tradein_fdw_reader" fi @@ -279,6 +273,21 @@ jobs: docker compose -p gendesign -f docker-compose.prod.yml up -d + # Defense: ensure postgres is in gendesign_shared network for tradein FDW. + # `compose up -d` should detect networks: shared addition and recreate + # postgres, but in PR #493 deploy/1156 incident the bootstrap step failed + # earlier so this code path never ran. Plus compose sometimes skips + # recreate if it thinks config is "compatible enough". Verify explicitly. + if ! docker inspect gendesign-postgres-1 \ + --format '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' \ + 2>/dev/null | grep -q gendesign_shared; then + echo "⚠️ postgres not in gendesign_shared after compose up — force-recreating" + docker compose -p gendesign -f docker-compose.prod.yml up -d \ + --force-recreate --no-deps postgres + # Brief settle window: backend will reconnect after postgres restart. + sleep 5 + fi + # backend/.env.runtime изменения (SENTRY_RELEASE, GLITCHTIP_DSN) # требуют --force-recreate — обычный `up -d` не перечитывает env_file # если только image не сменился. На deploy где меняется только runtime diff --git a/ops/db-bootstrap/set_tradein_fdw_password.sql b/ops/db-bootstrap/set_tradein_fdw_password.sql new file mode 100644 index 00000000..c0742ab2 --- /dev/null +++ b/ops/db-bootstrap/set_tradein_fdw_password.sql @@ -0,0 +1,27 @@ +-- Set tradein_fdw_reader password from env. +-- Applied by .forgejo/workflows/deploy.yml after main DB migrations: +-- psql -v pw="$GENDESIGN_FDW_PASSWORD" -f ops/db-bootstrap/set_tradein_fdw_password.sql +-- +-- Idempotent: ALTER if role exists, NOTICE and continue if missing +-- (migration 100_tradein_fdw_role.sql might not have applied yet on first deploy). +-- Password is NEVER stored in this file or in git — only the variable name. +-- +-- Format %L escapes the password as a SQL string literal — safe even with quotes. +-- Combined with the regex whitelist on the backend side (fdw.py _PASSWORD_RE) +-- this gives defense-in-depth. +-- +-- Extracted from deploy.yml inline DO block (PR #493 deploy/1156 incident). +-- The original block used multi-line backslash continuations inside a double-quoted +-- bash string with $$-dollar-quoting + psql variable substitution — fragile escaping +-- that likely caused the bootstrap step to fail before `compose up -d` ran. +-- Using `-f ` with psql-native variable substitution via `-v pw=` is safe. + +DO $$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'tradein_fdw_reader') THEN + EXECUTE format('ALTER ROLE tradein_fdw_reader WITH PASSWORD %L', :'pw'); + RAISE NOTICE 'tradein_fdw_reader password set'; + ELSE + RAISE NOTICE 'tradein_fdw_reader role missing — migration 100_tradein_fdw_role.sql not applied yet'; + END IF; +END $$;