103 lines
4.2 KiB
Bash
Executable file
103 lines
4.2 KiB
Bash
Executable file
#!/usr/bin/env bash
|
||
# Периодический скрейп всех площадок по 5 якорям ЕКБ.
|
||
# Запускается из cron на ПРОД-хосте (46.173.16.127).
|
||
# Запрос к backend идёт через `docker exec tradein-backend curl localhost:8000`
|
||
# — не зависит от Caddy и не выставляет admin/scrape наружу.
|
||
#
|
||
# Usage: ./cron-scrape.sh [all|avito|cian|yandex|n1] [deep|fast]
|
||
#
|
||
# Crontab пример:
|
||
# 0 */3 * * * /opt/gendesign/tradein-mvp/deploy/cron-scrape.sh all deep
|
||
# 30 * * * * /opt/gendesign/tradein-mvp/deploy/cron-scrape.sh all fast
|
||
|
||
set -euo pipefail
|
||
|
||
# cron запускается с ПУСТЫМ окружением. Подхватываем .env.runtime сами:
|
||
# он лежит на уровень выше скрипта — tradein-mvp/.env.runtime.
|
||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||
ENV_RUNTIME="${ENV_RUNTIME:-$SCRIPT_DIR/../.env.runtime}"
|
||
if [[ -f "$ENV_RUNTIME" ]]; then
|
||
set -a
|
||
# shellcheck source=/dev/null
|
||
source "$ENV_RUNTIME"
|
||
set +a
|
||
fi
|
||
|
||
SOURCE="${1:-all}"
|
||
MODE="${2:-fast}"
|
||
CONTAINER="${TRADEIN_BACKEND_CONTAINER:-tradein-backend}"
|
||
LOG="${LOG:-/var/log/tradein-scrape.log}"
|
||
|
||
log() {
|
||
echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"
|
||
}
|
||
|
||
# 5 якорей по районам ЕКБ. lat lon name
|
||
ANCHORS=(
|
||
"56.8400 60.6050 Центр"
|
||
"56.7950 60.5300 ЮЗ"
|
||
"56.8970 60.6100 Уралмаш"
|
||
"56.7700 60.5500 Академический"
|
||
"56.8650 60.6200 Пионерский"
|
||
)
|
||
|
||
build_payload() {
|
||
local lat="$1" lon="$2" src="$3" mode="$4"
|
||
local sources_json
|
||
case "$src" in
|
||
yandex) sources_json='["yandex"]' ;;
|
||
avito) sources_json='["avito"]' ;;
|
||
cian) sources_json='["cian"]' ;;
|
||
n1) sources_json='["n1"]' ;;
|
||
all|*) sources_json='["avito","cian","yandex","n1"]' ;;
|
||
esac
|
||
if [[ "$mode" == "deep" ]]; then
|
||
echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"deep_yandex\":true,\"multi_room_cian\":true,\"multi_room_n1\":true}"
|
||
else
|
||
echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"multi_room_cian\":true,\"multi_room_n1\":true}"
|
||
fi
|
||
}
|
||
|
||
log "=== cron-scrape start: source=$SOURCE mode=$MODE container=$CONTAINER ==="
|
||
|
||
for anchor in "${ANCHORS[@]}"; do
|
||
read -r lat lon name <<< "$anchor"
|
||
payload=$(build_payload "$lat" "$lon" "$SOURCE" "$MODE")
|
||
log "→ $name ($lat, $lon)"
|
||
# curl выполняется ВНУТРИ backend-контейнера → localhost:8000, без публичного доступа.
|
||
resp=$(docker exec "$CONTAINER" curl -s -X POST \
|
||
http://localhost:8000/api/v1/admin/scrape \
|
||
-H "Content-Type: application/json" \
|
||
-d "$payload" \
|
||
-m 600 2>/dev/null || echo '{"error":"exec_or_timeout"}')
|
||
log "← $name $resp"
|
||
sleep 5
|
||
done
|
||
|
||
# Геокодинг свежих лотов без координат (Cian/N1 отдают только адрес).
|
||
# Чанками по 100 (~200с/чанк, под таймаутом), пока remaining > 0.
|
||
# Макс 15 чанков = 1500 адресов. geocode_tried_at не даёт зациклиться.
|
||
log "→ geocode-missing listings (чанками)"
|
||
for gi in $(seq 1 15); do
|
||
geo=$(docker exec "$CONTAINER" curl -s -X POST \
|
||
"http://localhost:8000/api/v1/admin/geocode-missing?limit=100&target=listings" \
|
||
-m 320 2>/dev/null || echo '{"error":"geocode_timeout","remaining":0}')
|
||
log "← listings чанк $gi: $geo"
|
||
remaining=$(echo "$geo" | grep -oE '"remaining":[0-9]+' | grep -oE '[0-9]+' || echo 0)
|
||
[ "${remaining:-0}" -eq 0 ] && break
|
||
sleep 2
|
||
done
|
||
|
||
# Сделки Росреестра тоже без координат (импорт отдаёт только улицу).
|
||
log "→ geocode-missing deals (чанками)"
|
||
for gi in $(seq 1 30); do
|
||
geo=$(docker exec "$CONTAINER" curl -s -X POST \
|
||
"http://localhost:8000/api/v1/admin/geocode-missing?limit=100&target=deals" \
|
||
-m 320 2>/dev/null || echo '{"error":"geocode_timeout","remaining":0}')
|
||
log "← deals чанк $gi: $geo"
|
||
remaining=$(echo "$geo" | grep -oE '"remaining":[0-9]+' | grep -oE '[0-9]+' || echo 0)
|
||
[ "${remaining:-0}" -eq 0 ] && break
|
||
sleep 2
|
||
done
|
||
|
||
log "=== cron-scrape done ==="
|