bot-backend
72a4c122c4
Причина: docker restart tradein-backend (при каждом API-деплое) убивал бегущие sweep'ы avito/cian/rosreestr — они могут идти часами, и их прерывание середине батча создавало zombie-runs и пропуски данных. Что сделано: - scheduler_main.py — standalone entrypoint для tradein-scraper; SIGTERM/SIGINT отменяют scheduler_loop чисто; GlitchTip init без Starlette/FastAPI интеграций. - docker-compose.prod.yml — новый сервис scraper (тот же image, команда python -m app.scheduler_main, только tradein-net); SCHEDULER_ENABLE=false в backend. - deploy-tradein.yml — селективный up -d --no-deps: scraper пересоздаётся только при изменениях scraper/infra путей (SCRAPER_CHANGED), не при каждом backend-деплое. - tests/test_scheduler_main.py — импорт, kill-switch _should_run(), чистая отмена _run().
353 lines
17 KiB
YAML
353 lines
17 KiB
YAML
name: Deploy Trade-In
|
||
|
||
# Forgejo Actions — отдельный pipeline для подпроекта tradein-mvp/.
|
||
# Триггерится только на изменения внутри tradein-mvp/ (или этого workflow),
|
||
# не пересекается с основным deploy.yml.
|
||
on:
|
||
push:
|
||
branches: [main]
|
||
paths:
|
||
- "tradein-mvp/**"
|
||
- ".forgejo/workflows/deploy-tradein.yml"
|
||
workflow_dispatch:
|
||
|
||
concurrency:
|
||
group: deploy-tradein-prod
|
||
cancel-in-progress: false
|
||
|
||
env:
|
||
IMAGE_BACKEND: ghcr.io/lekss361/gendesign-tradein-backend
|
||
IMAGE_FRONTEND: ghcr.io/lekss361/gendesign-tradein-frontend
|
||
IMAGE_BROWSER: ghcr.io/lekss361/gendesign-tradein-browser
|
||
|
||
jobs:
|
||
changes:
|
||
runs-on: ubuntu-latest
|
||
outputs:
|
||
backend: ${{ steps.filter.outputs.backend }}
|
||
frontend: ${{ steps.filter.outputs.frontend }}
|
||
browser: ${{ steps.filter.outputs.browser }}
|
||
infra: ${{ steps.filter.outputs.infra }}
|
||
scraper: ${{ steps.filter.outputs.scraper }}
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
- uses: dorny/paths-filter@v3
|
||
id: filter
|
||
with:
|
||
filters: |
|
||
backend:
|
||
- 'tradein-mvp/backend/**'
|
||
frontend:
|
||
- 'tradein-mvp/frontend/**'
|
||
browser:
|
||
- 'tradein-mvp/browser/**'
|
||
infra:
|
||
- 'tradein-mvp/docker-compose.prod.yml'
|
||
- 'tradein-mvp/deploy/**'
|
||
- '.forgejo/workflows/deploy-tradein.yml'
|
||
scraper:
|
||
- 'tradein-mvp/backend/app/services/scrapers/**'
|
||
- 'tradein-mvp/backend/app/services/scheduler.py'
|
||
- 'tradein-mvp/backend/app/scheduler_main.py'
|
||
- 'tradein-mvp/backend/app/tasks/**'
|
||
|
||
# Quality gate: pytest MUST pass before any image is built/deployed (#666).
|
||
# Runs the tradein-mvp/backend suite; a red test blocks build + deploy.
|
||
# Tests use mocks + a stub DATABASE_URL — no real Postgres/Redis needed.
|
||
# 2 pre-existing order-dependent tests are deselected (see DESELECT note below).
|
||
test:
|
||
runs-on: ubuntu-latest
|
||
needs: changes
|
||
if: |
|
||
needs.changes.outputs.backend == 'true' ||
|
||
needs.changes.outputs.infra == 'true' ||
|
||
github.event_name == 'workflow_dispatch'
|
||
defaults:
|
||
run:
|
||
working-directory: ./tradein-mvp/backend
|
||
env:
|
||
# psycopg v3 requires a parseable URL at import time; never connected to.
|
||
DATABASE_URL: postgresql+psycopg://test:test@localhost:5432/test
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Install uv
|
||
# Официальный standalone-инсталлер: системный `pip install uv` на
|
||
# ubuntu-runner падает с PEP 668 externally-managed-environment (#666 CI).
|
||
run: |
|
||
curl -LsSf https://astral.sh/uv/install.sh | sh
|
||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||
|
||
- name: Sync deps (incl. dev group — pytest)
|
||
# NB: tradein-mvp/backend/uv.lock is gitignored (see tradein-mvp/.gitignore),
|
||
# so --frozen can't work in CI (no lockfile in a fresh checkout). Resolve
|
||
# from pyproject.toml like the Dockerfile does (uv sync --no-dev there).
|
||
run: uv sync
|
||
|
||
- name: Run pytest (tradein-mvp/backend)
|
||
# DESELECT (2026-05): two pre-existing tests fail only in whole-suite
|
||
# ordering (pass in isolation) due to global state leak from other test
|
||
# modules — not introduced here. Excluded so the gate is reliably green;
|
||
# tracked separately. Everything else (1087 tests) must pass.
|
||
run: |
|
||
uv run pytest -q \
|
||
--deselect "tests/test_search_api.py::test_search_cache_hit" \
|
||
--deselect "tests/test_cian_valuation.py::test_cache_hit_returns_cached"
|
||
|
||
build-backend:
|
||
runs-on: ubuntu-latest
|
||
needs: [changes, test]
|
||
if: |
|
||
needs.changes.outputs.backend == 'true' ||
|
||
needs.changes.outputs.infra == 'true' ||
|
||
github.event_name == 'workflow_dispatch'
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions)
|
||
env:
|
||
GHCR_PAT: ${{ secrets.GHCR_PAT }}
|
||
run: |
|
||
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
|
||
|
||
- name: Set up Docker Buildx
|
||
uses: docker/setup-buildx-action@v3
|
||
|
||
- name: Build & push tradein-backend
|
||
uses: docker/build-push-action@v6
|
||
with:
|
||
context: ./tradein-mvp/backend
|
||
push: true
|
||
cache-from: type=registry,ref=${{ env.IMAGE_BACKEND }}:buildcache
|
||
cache-to: type=registry,ref=${{ env.IMAGE_BACKEND }}:buildcache,mode=max
|
||
tags: |
|
||
${{ env.IMAGE_BACKEND }}:latest
|
||
${{ env.IMAGE_BACKEND }}:${{ github.sha }}
|
||
|
||
build-frontend:
|
||
runs-on: ubuntu-latest
|
||
needs: changes
|
||
if: |
|
||
needs.changes.outputs.frontend == 'true' ||
|
||
needs.changes.outputs.infra == 'true' ||
|
||
github.event_name == 'workflow_dispatch'
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions)
|
||
env:
|
||
GHCR_PAT: ${{ secrets.GHCR_PAT }}
|
||
run: |
|
||
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
|
||
|
||
- name: Set up Docker Buildx
|
||
uses: docker/setup-buildx-action@v3
|
||
|
||
- name: Build & push tradein-frontend
|
||
uses: docker/build-push-action@v6
|
||
with:
|
||
context: ./tradein-mvp/frontend
|
||
push: true
|
||
# basePath=/trade-in baked-in во время build (Next.js)
|
||
build-args: |
|
||
NEXT_PUBLIC_BASE_PATH=/trade-in
|
||
NEXT_PUBLIC_API_BASE_URL=/trade-in
|
||
NEXT_PUBLIC_ENABLE_PREVIEW=1
|
||
cache-from: type=registry,ref=${{ env.IMAGE_FRONTEND }}:buildcache
|
||
cache-to: type=registry,ref=${{ env.IMAGE_FRONTEND }}:buildcache,mode=max
|
||
tags: |
|
||
${{ env.IMAGE_FRONTEND }}:latest
|
||
${{ env.IMAGE_FRONTEND }}:${{ github.sha }}
|
||
|
||
build-browser:
|
||
runs-on: ubuntu-latest
|
||
needs: changes
|
||
# tradein-browser несёт camoufox + Firefox-build (#905). Триггерится на
|
||
# изменения browser/ или infra (compose ссылается на образ) или вручную.
|
||
if: |
|
||
needs.changes.outputs.browser == 'true' ||
|
||
needs.changes.outputs.infra == 'true' ||
|
||
github.event_name == 'workflow_dispatch'
|
||
steps:
|
||
- uses: actions/checkout@v4
|
||
|
||
- name: Login to GHCR (shell-based — docker/login-action@v3 unreliable под Forgejo Actions)
|
||
env:
|
||
GHCR_PAT: ${{ secrets.GHCR_PAT }}
|
||
run: |
|
||
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
|
||
|
||
- name: Set up Docker Buildx
|
||
uses: docker/setup-buildx-action@v3
|
||
|
||
- name: Build & push tradein-browser
|
||
uses: docker/build-push-action@v6
|
||
with:
|
||
context: ./tradein-mvp/browser
|
||
push: true
|
||
cache-from: type=registry,ref=${{ env.IMAGE_BROWSER }}:buildcache
|
||
cache-to: type=registry,ref=${{ env.IMAGE_BROWSER }}:buildcache,mode=max
|
||
tags: |
|
||
${{ env.IMAGE_BROWSER }}:latest
|
||
${{ env.IMAGE_BROWSER }}:${{ github.sha }}
|
||
|
||
deploy:
|
||
runs-on: ubuntu-latest
|
||
needs: [changes, test, build-backend, build-frontend, build-browser]
|
||
# NB: a failed `test` skips build-backend (result='skipped', not 'failure'),
|
||
# so we must block deploy on test failure explicitly (#666 quality gate).
|
||
if: |
|
||
always() &&
|
||
!cancelled() &&
|
||
needs.test.result != 'failure' &&
|
||
needs.build-backend.result != 'failure' &&
|
||
needs.build-frontend.result != 'failure' &&
|
||
needs.build-browser.result != 'failure'
|
||
steps:
|
||
- name: Deploy via SSH
|
||
uses: appleboy/ssh-action@v1.0.3
|
||
env:
|
||
IMAGE_TAG: latest
|
||
GHCR_PAT: ${{ secrets.GHCR_PAT }}
|
||
SCRAPER_CHANGED: ${{ needs.changes.outputs.scraper == 'true' || needs.changes.outputs.infra == 'true' || github.event_name == 'workflow_dispatch' }}
|
||
with:
|
||
host: ${{ secrets.DEPLOY_HOST }}
|
||
username: ${{ secrets.DEPLOY_USER }}
|
||
key: ${{ secrets.DEPLOY_SSH_KEY }}
|
||
port: ${{ secrets.DEPLOY_PORT }}
|
||
envs: IMAGE_TAG,GHCR_PAT,SCRAPER_CHANGED
|
||
script: |
|
||
set -euo pipefail
|
||
cd /opt/gendesign
|
||
|
||
# repo уже clone'ен — origin = Forgejo. Подтягиваем последний main.
|
||
git fetch origin main
|
||
git reset --hard origin/main
|
||
|
||
cd tradein-mvp
|
||
|
||
# .env.runtime создаётся вручную при первом запуске (см. README-АДМИНУ.md).
|
||
# Здесь только подгружаем переменные для docker compose (POSTGRES_PASSWORD).
|
||
if [ ! -f .env.runtime ]; then
|
||
echo "ERROR: /opt/gendesign/tradein-mvp/.env.runtime отсутствует."
|
||
echo "Создай его вручную (см. tradein-mvp/README.md или DEPLOY.md)."
|
||
exit 1
|
||
fi
|
||
chmod 600 .env.runtime
|
||
set -a; source .env.runtime; set +a
|
||
|
||
# External network для Caddy (он в основном gendesign-стеке)
|
||
docker network inspect gendesign_shared >/dev/null 2>&1 \
|
||
|| docker network create gendesign_shared
|
||
|
||
# Re-login to GHCR (PAT может быть rotated)
|
||
echo "$GHCR_PAT" | docker login ghcr.io -u lekss361 --password-stdin
|
||
|
||
export IMAGE_TAG="$IMAGE_TAG"
|
||
docker compose -p gendesign-tradein -f docker-compose.prod.yml pull
|
||
|
||
# Селективный up: scraper НЕ пересоздаём на каждый backend-деплой (#1182) —
|
||
# один image :latest на backend+scraper, блочный up -d рестартовал бы scraper
|
||
# и убивал бегущий sweep. Scraper поднимается только при scraper/infra изменениях.
|
||
docker compose -p gendesign-tradein -f docker-compose.prod.yml up -d --no-deps postgres browser backend frontend
|
||
if [ "${SCRAPER_CHANGED:-true}" = "true" ]; then
|
||
echo "→ scraper paths changed — recreating tradein-scraper"
|
||
docker compose -p gendesign-tradein -f docker-compose.prod.yml up -d --no-deps scraper
|
||
else
|
||
echo "→ scraper unchanged — tradein-scraper left running (подхватит новый image при следующем своём рестарте)"
|
||
fi
|
||
|
||
# Применяем SQL миграции (если есть backend/data/sql/*.sql)
|
||
# Postgres init load *.sql из /docker-entrypoint-initdb.d ТОЛЬКО при первом
|
||
# старте volume. Здесь — для повторных миграций после первого запуска.
|
||
# Tracking через _schema_migrations (порт паттерна из deploy.yml):
|
||
# каждый .sql применяется РОВНО один раз, failed migration → exit 1
|
||
# (никаких swallowed errors). cwd = /opt/gendesign/tradein-mvp.
|
||
sleep 5
|
||
|
||
# Pre-existence detection ДО CREATE TABLE: если таблицы ещё нет, это
|
||
# первый deploy после внедрения tracking на уже-наполненной prod-БД
|
||
# (миграции 001-076 живут в схеме). 7 из них (002/003/052/053/054/063/072)
|
||
# содержат INSERT-seed БЕЗ ON CONFLICT — повторный прогон под строгим
|
||
# ON_ERROR_STOP упал бы на PK violation. Поэтому: baseline (см. ниже).
|
||
migrations_table_existed=$(docker compose -p gendesign-tradein -f docker-compose.prod.yml exec -T postgres \
|
||
psql -U "${TRADEIN_POSTGRES_USER:-tradein}" -d tradein -tAc \
|
||
"SELECT to_regclass('public._schema_migrations') IS NOT NULL;" | tr -d '[:space:]')
|
||
|
||
docker compose -p gendesign-tradein -f docker-compose.prod.yml exec -T postgres \
|
||
psql -U "${TRADEIN_POSTGRES_USER:-tradein}" -d tradein -v ON_ERROR_STOP=on -c "
|
||
CREATE TABLE IF NOT EXISTS _schema_migrations (
|
||
filename TEXT PRIMARY KEY,
|
||
applied_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||
);
|
||
"
|
||
|
||
if [ "$migrations_table_existed" != "t" ]; then
|
||
# BASELINE: таблицы не было → seed ВСЕ текущие миграции как applied
|
||
# БЕЗ их прогона. prod уже работает на этой схеме; помечаем её
|
||
# текущим состоянием, чтобы под строгий gate попадали только НОВЫЕ
|
||
# (077+) миграции. INSERT ... ON CONFLICT DO NOTHING — идемпотентно.
|
||
echo "→ _schema_migrations отсутствовала — baseline существующих миграций (без прогона)"
|
||
for sql_file in $(ls -1 backend/data/sql/*.sql 2>/dev/null | sort); do
|
||
fname=$(basename "$sql_file")
|
||
echo " baseline: $fname"
|
||
docker compose -p gendesign-tradein -f docker-compose.prod.yml exec -T postgres \
|
||
psql -U "${TRADEIN_POSTGRES_USER:-tradein}" -d tradein -v ON_ERROR_STOP=on -c \
|
||
"INSERT INTO _schema_migrations (filename) VALUES ('$fname') ON CONFLICT DO NOTHING;"
|
||
done
|
||
echo "Baseline complete — existing schema marked as applied."
|
||
fi
|
||
|
||
for sql_file in $(ls -1 backend/data/sql/*.sql 2>/dev/null | sort); do
|
||
fname=$(basename "$sql_file")
|
||
applied=$(docker compose -p gendesign-tradein -f docker-compose.prod.yml exec -T postgres \
|
||
psql -U "${TRADEIN_POSTGRES_USER:-tradein}" -d tradein -tAc \
|
||
"SELECT COUNT(*) FROM _schema_migrations WHERE filename='$fname'" | tr -d '[:space:]')
|
||
if [ "$applied" = "0" ]; then
|
||
echo "→ Applying migration: $fname"
|
||
docker compose -p gendesign-tradein -f docker-compose.prod.yml exec -T postgres \
|
||
psql -U "${TRADEIN_POSTGRES_USER:-tradein}" -d tradein -v ON_ERROR_STOP=on \
|
||
< "$sql_file" \
|
||
|| { echo "FAILED on migration: $fname"; exit 1; }
|
||
docker compose -p gendesign-tradein -f docker-compose.prod.yml exec -T postgres \
|
||
psql -U "${TRADEIN_POSTGRES_USER:-tradein}" -d tradein -c \
|
||
"INSERT INTO _schema_migrations (filename) VALUES ('$fname') ON CONFLICT DO NOTHING;"
|
||
else
|
||
echo "✓ Already applied: $fname"
|
||
fi
|
||
done
|
||
echo "All migrations applied."
|
||
|
||
# Retry backend lifespan hook AFTER migrations applied.
|
||
# tradein-backend startup runs ensure_fdw_user_mapping which needs
|
||
# FOREIGN SERVER gendesign_remote (created by 060_postgres_fdw_extension.sql).
|
||
# Without restart, the first compose-up's startup hook failed with
|
||
# "server gendesign_remote does not exist" because migrations hadn't run yet.
|
||
# See PR #493 deploy/1156 for the incident details.
|
||
echo "→ Restarting tradein-backend so lifespan hook retries USER MAPPING setup"
|
||
docker restart tradein-backend
|
||
# Give backend time to come up before Caddy reload + health check below
|
||
sleep 5
|
||
|
||
# Caddy reload — основной Caddyfile содержит inline tradein routes
|
||
# (см. Caddyfile в корне репы). Reload, чтобы Caddy перечитал DNS
|
||
# tradein-backend / tradein-frontend (они в gendesign_shared network).
|
||
cd /opt/gendesign
|
||
docker compose -p gendesign -f docker-compose.prod.yml exec -T caddy \
|
||
caddy reload --config /etc/caddy/Caddyfile || true
|
||
|
||
# Health check
|
||
for i in $(seq 1 30); do
|
||
docker compose -p gendesign-tradein -f /opt/gendesign/tradein-mvp/docker-compose.prod.yml \
|
||
exec -T backend curl -fsS http://localhost:8000/health && break
|
||
sleep 1
|
||
done
|
||
|
||
# Cleanup старых образов
|
||
for repo in ghcr.io/lekss361/gendesign-tradein-backend \
|
||
ghcr.io/lekss361/gendesign-tradein-frontend; do
|
||
docker images "$repo" --format '{{.Repository}}:{{.Tag}}' \
|
||
| grep -v ':latest$' | tail -n +3 \
|
||
| xargs -r docker rmi 2>/dev/null || true
|
||
done
|
||
docker image prune -af || true
|