42a0c31f27
Добавлен 4-й источник N1.ru (ekaterinburg.n1.ru) — региональный портал, хорошо представлен на Урале. DOM-парсинг карточек data-test=offers-list-item через curl_cffi. Multi-room сегментация 1/2/3/4к. cron-scrape.sh переписан под прод: запрос к backend через docker exec (без публичного admin/scrape), все 4 источника (avito/cian/yandex/n1).
66 lines
2.4 KiB
Bash
Executable file
66 lines
2.4 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Периодический скрейп всех площадок по 5 якорям ЕКБ.
|
|
# Запускается из cron на ПРОД-хосте (46.173.16.127).
|
|
# Запрос к backend идёт через `docker exec tradein-backend curl localhost:8000`
|
|
# — не зависит от Caddy и не выставляет admin/scrape наружу.
|
|
#
|
|
# Usage: ./cron-scrape.sh [all|avito|cian|yandex|n1] [deep|fast]
|
|
#
|
|
# Crontab пример:
|
|
# 0 */3 * * * /opt/gendesign/tradein-mvp/deploy/cron-scrape.sh all deep
|
|
# 30 * * * * /opt/gendesign/tradein-mvp/deploy/cron-scrape.sh all fast
|
|
|
|
set -euo pipefail
|
|
|
|
SOURCE="${1:-all}"
|
|
MODE="${2:-fast}"
|
|
CONTAINER="${TRADEIN_BACKEND_CONTAINER:-tradein-backend}"
|
|
LOG="${LOG:-/var/log/tradein-scrape.log}"
|
|
|
|
log() {
|
|
echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"
|
|
}
|
|
|
|
# 5 якорей по районам ЕКБ. lat lon name
|
|
ANCHORS=(
|
|
"56.8400 60.6050 Центр"
|
|
"56.7950 60.5300 ЮЗ"
|
|
"56.8970 60.6100 Уралмаш"
|
|
"56.7700 60.5500 Академический"
|
|
"56.8650 60.6200 Пионерский"
|
|
)
|
|
|
|
build_payload() {
|
|
local lat="$1" lon="$2" src="$3" mode="$4"
|
|
local sources_json
|
|
case "$src" in
|
|
yandex) sources_json='["yandex"]' ;;
|
|
avito) sources_json='["avito"]' ;;
|
|
cian) sources_json='["cian"]' ;;
|
|
n1) sources_json='["n1"]' ;;
|
|
all|*) sources_json='["avito","cian","yandex","n1"]' ;;
|
|
esac
|
|
if [[ "$mode" == "deep" ]]; then
|
|
echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"deep_yandex\":true,\"multi_room_cian\":true,\"multi_room_n1\":true}"
|
|
else
|
|
echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"multi_room_cian\":true,\"multi_room_n1\":true}"
|
|
fi
|
|
}
|
|
|
|
log "=== cron-scrape start: source=$SOURCE mode=$MODE container=$CONTAINER ==="
|
|
|
|
for anchor in "${ANCHORS[@]}"; do
|
|
read -r lat lon name <<< "$anchor"
|
|
payload=$(build_payload "$lat" "$lon" "$SOURCE" "$MODE")
|
|
log "→ $name ($lat, $lon)"
|
|
# curl выполняется ВНУТРИ backend-контейнера → localhost:8000, без публичного доступа.
|
|
resp=$(docker exec "$CONTAINER" curl -s -X POST \
|
|
http://localhost:8000/api/v1/admin/scrape \
|
|
-H "Content-Type: application/json" \
|
|
-d "$payload" \
|
|
-m 600 2>/dev/null || echo '{"error":"exec_or_timeout"}')
|
|
log "← $name $resp"
|
|
sleep 5
|
|
done
|
|
|
|
log "=== cron-scrape done ==="
|