#!/usr/bin/env bash # Avito city sweep — full ЕКБ pagination + enrichment. # Random delay 0-3h перед запуском чтобы cron не бил Avito в одно и то же время каждый день. # # Crontab: 0 2 * * * /opt/gendesign/tradein-mvp/deploy/avito-city-sweep.sh # Реальный запуск: 02:00-05:00 UTC случайно каждый день. # # Env overrides: # SWEEP_PAGES=3 — страниц на anchor (default 3) # SWEEP_DETAIL_TOP_N=20 — detail enrichment top-N (default 20) # SWEEP_REQUEST_DELAY=7.0 — задержка между запросами в секундах (default 7.0) # LOG=/var/log/tradein-avito-city-sweep.log # TRADEIN_BACKEND_CONTAINER=tradein-backend set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" LOG="${LOG:-/var/log/tradein-avito-city-sweep.log}" CONTAINER="${TRADEIN_BACKEND_CONTAINER:-tradein-backend}" log() { echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG" } # Random delay 0-10800 sec (0-3h). RANDOM gives 0..32767, mod 10800 → uniform 0..3h. DELAY=$((RANDOM % 10800)) log "=== avito-city-sweep: sleeping ${DELAY}s before start ===" sleep "$DELAY" # Параметры sweep (overrideable via env) PAGES_PER_ANCHOR="${SWEEP_PAGES:-3}" DETAIL_TOP_N="${SWEEP_DETAIL_TOP_N:-20}" REQUEST_DELAY="${SWEEP_REQUEST_DELAY:-7.0}" PAYLOAD=$(cat </dev/null || echo '{"error":"trigger_failed"}') log "<- response: $resp" # Проверка что run_id вернулся if echo "$resp" | grep -q '"run_id"'; then RUN_ID=$(echo "$resp" | grep -o '"run_id":[0-9]*' | grep -o '[0-9]*') log "=== avito-city-sweep: started run_id=${RUN_ID} (background task на backend)" log "=== Polling: GET /api/v1/admin/scrape/avito-city-sweep/runs" else log "=== avito-city-sweep: WARNING — unexpected response, check backend logs ===" exit 1 fi