66 lines
2.4 KiB
Bash
66 lines
2.4 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
API="${TRADEIN_API_URL:-http://localhost:8000}"
|
|
BATCH="${BATCH_SIZE:-100}"
|
|
MAX_ITERS="${MAX_ITERS:-1000}"
|
|
DO_LISTINGS="${DO_LISTINGS:-true}"
|
|
DO_VALUATIONS="${DO_VALUATIONS:-true}"
|
|
DO_HOUSES="${DO_HOUSES:-true}" # enabled after migration 071_houses_cian_zhk_url.sql (#568)
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Cian backfill runner — drains offer_price_history + external_valuations backlog.
|
|
|
|
Usage:
|
|
$0 [--dry-run]
|
|
|
|
Env vars:
|
|
TRADEIN_API_URL default: http://localhost:8000
|
|
BATCH_SIZE default: 100 (max 200 per endpoint cap)
|
|
MAX_ITERS default: 1000 (safety stop)
|
|
DO_LISTINGS default: true
|
|
DO_VALUATIONS default: true
|
|
DO_HOUSES default: true (requires migration 071_houses_cian_zhk_url.sql on prod)
|
|
|
|
Examples:
|
|
$0 --dry-run # count what would be processed
|
|
BATCH_SIZE=50 $0 # drain at half pace
|
|
DO_VALUATIONS=false $0 # only history
|
|
TRADEIN_API_URL=https://gendsgn.ru/api $0
|
|
EOF
|
|
}
|
|
|
|
DRY=""
|
|
if [[ "${1:-}" == "--dry-run" ]]; then DRY="&dry_run=true"; fi
|
|
if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then usage; exit 0; fi
|
|
|
|
echo "Cian backfill: API=$API BATCH=$BATCH listings=$DO_LISTINGS valuations=$DO_VALUATIONS houses=$DO_HOUSES dry=${DRY:-no}"
|
|
|
|
cum_listings=0
|
|
cum_valuations=0
|
|
i=0
|
|
while (( i < MAX_ITERS )); do
|
|
i=$((i+1))
|
|
url="$API/api/v1/admin/scrape/cian-backfill-history?batch_size=$BATCH&do_listings=$DO_LISTINGS&do_valuations=$DO_VALUATIONS&do_houses=$DO_HOUSES${DRY}"
|
|
resp=$(curl -fsS -X POST "$url")
|
|
lt=$(echo "$resp" | jq -r '.listings_total // 0')
|
|
vt=$(echo "$resp" | jq -r '.valuations_total // 0')
|
|
ht=$(echo "$resp" | jq -r '.houses_total // 0')
|
|
ls=$(echo "$resp" | jq -r '.listings_succeeded // 0')
|
|
vs=$(echo "$resp" | jq -r '.valuations_succeeded // 0')
|
|
hs=$(echo "$resp" | jq -r '.houses_succeeded // 0')
|
|
dur=$(echo "$resp" | jq -r '.duration_sec // 0')
|
|
cum_listings=$((cum_listings + ls))
|
|
cum_valuations=$((cum_valuations + vs))
|
|
echo "[$i] listings=$lt (ok=$ls) valuations=$vt (ok=$vs) houses=$ht (ok=$hs) ${dur}s cum: hist=$cum_listings val=$cum_valuations"
|
|
if (( lt == 0 && vt == 0 && ht == 0 )); then
|
|
echo "Drained — stopping."
|
|
break
|
|
fi
|
|
if [[ -n "$DRY" ]]; then
|
|
echo "Dry-run — no DB writes; stopping after first probe."
|
|
break
|
|
fi
|
|
done
|
|
echo "Done. Total: history=$cum_listings valuations=$cum_valuations iterations=$i"
|