#!/usr/bin/env bash
# Периодический скрейп Yandex+Avito по 5 якорям ЕКБ.
# Запускается из cron. Логи в /var/log/tradein-scrape.log
#
# Usage: ./cron-scrape.sh [yandex|avito|all] [deep|fast]
#
# Примеры в crontab:
#   # Yandex deep раз в 3 часа
#   0 */3 * * * /opt/tradein-mvp/deploy/cron-scrape.sh yandex deep
#   # Avito быстро каждый час
#   30 * * * * /opt/tradein-mvp/deploy/cron-scrape.sh avito fast

set -euo pipefail

SOURCE="${1:-all}"
MODE="${2:-fast}"
API="${API:-http://127.0.0.1:8091}"
LOG="${LOG:-/var/log/tradein-scrape.log}"

log() {
  echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"
}

# 5 якорей по районам ЕКБ. lat lon name
# Для Yandex DATE_DESC возвращает одинаковую выдачу регардлесс — но Cian/Avito
# реально фильтруют по координатам, так что multi-anchor нужен для них.
ANCHORS=(
  "56.8400 60.6050 Центр"
  "56.7950 60.5300 ЮЗ"
  "56.8970 60.6100 Уралмаш"
  "56.7700 60.5500 Академический"
  "56.8650 60.6200 Пионерский"
)

build_payload() {
  local lat="$1" lon="$2" src="$3" mode="$4"
  local sources_json
  case "$src" in
    yandex) sources_json='["yandex"]' ;;
    avito)  sources_json='["avito"]' ;;
    cian)   sources_json='["cian"]' ;;
    all)    sources_json='["avito","cian","yandex"]' ;;
    *)      sources_json='["avito","cian","yandex"]' ;;
  esac
  if [[ "$mode" == "deep" ]]; then
    echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"deep_yandex\":true,\"multi_room_cian\":true}"
  else
    echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"multi_room_cian\":true}"
  fi
}

log "=== cron-scrape start: source=$SOURCE mode=$MODE ==="

# Circuit breaker: если Avito IP-rate-limited (3 нуля подряд) — пропускаем оставшиеся.
zero_fetch_streak=0
for anchor in "${ANCHORS[@]}"; do
  if [[ "$zero_fetch_streak" -ge 3 ]]; then
    log "⚠ circuit-breaker tripped (3× zero-fetch in a row) — skipping remaining anchors"
    break
  fi
  read -r lat lon name <<< "$anchor"
  payload=$(build_payload "$lat" "$lon" "$SOURCE" "$MODE")
  log "→ $name ($lat, $lon) payload=$payload"
  resp=$(curl -s -X POST "$API/api/v1/admin/scrape" \
    -H "Content-Type: application/json" \
    -d "$payload" \
    -m 300 || echo '{"error":"curl_timeout"}')
  log "← $name $resp"
  # Парсим total_fetched — если 0, инкремент streak; иначе reset
  fetched=$(echo "$resp" | grep -oE '"total_fetched":[0-9]+' | grep -oE '[0-9]+' || echo 0)
  if [[ "${fetched:-0}" == "0" ]]; then
    zero_fetch_streak=$((zero_fetch_streak + 1))
  else
    zero_fetch_streak=0
  fi
  sleep 5
done

log "=== cron-scrape done ==="