02267d0306
- FastAPI backend: PostGIS estimator + 3 scrapers (Avito/Cian/Yandex)
- Next.js 15 frontend: tradein.html mockup design, basePath=/trade-in
- WeasyPrint PDF (Брусника-style 4-page report)
- Address autocomplete с typo-tolerance + 6 EKB presets
- Изолированный docker stack gendesign-tradein (отдельная postgres БД)
- Caddy inline routes: gendsgn.ru/trade-in/* и /trade-in/api/v1/*
- Forgejo Actions: .forgejo/workflows/deploy-tradein.yml (shell-based GHCR login)
- Триггер только по paths: tradein-mvp/** (не пересекается с deploy.yml)
- Образы: ghcr.io/lekss361/gendesign-tradein-{backend,frontend}:latest
Первый запуск на сервере (вручную, один раз):
- создать /opt/gendesign/tradein-mvp/.env.runtime (postgres pwd, contact email)
- docker network create gendesign_shared (если нет)
- docker compose -p gendesign-tradein up -d
- docker compose -p gendesign exec caddy caddy reload
79 lines
2.9 KiB
Bash
79 lines
2.9 KiB
Bash
#!/usr/bin/env bash
|
||
# Периодический скрейп Yandex+Avito по 5 якорям ЕКБ.
|
||
# Запускается из cron. Логи в /var/log/tradein-scrape.log
|
||
#
|
||
# Usage: ./cron-scrape.sh [yandex|avito|all] [deep|fast]
|
||
#
|
||
# Примеры в crontab:
|
||
# # Yandex deep раз в 3 часа
|
||
# 0 */3 * * * /opt/tradein-mvp/deploy/cron-scrape.sh yandex deep
|
||
# # Avito быстро каждый час
|
||
# 30 * * * * /opt/tradein-mvp/deploy/cron-scrape.sh avito fast
|
||
|
||
set -euo pipefail
|
||
|
||
SOURCE="${1:-all}"
|
||
MODE="${2:-fast}"
|
||
API="${API:-http://127.0.0.1:8091}"
|
||
LOG="${LOG:-/var/log/tradein-scrape.log}"
|
||
|
||
log() {
|
||
echo "[$(date -u +'%Y-%m-%dT%H:%M:%SZ')] $*" | tee -a "$LOG"
|
||
}
|
||
|
||
# 5 якорей по районам ЕКБ. lat lon name
|
||
# Для Yandex DATE_DESC возвращает одинаковую выдачу регардлесс — но Cian/Avito
|
||
# реально фильтруют по координатам, так что multi-anchor нужен для них.
|
||
ANCHORS=(
|
||
"56.8400 60.6050 Центр"
|
||
"56.7950 60.5300 ЮЗ"
|
||
"56.8970 60.6100 Уралмаш"
|
||
"56.7700 60.5500 Академический"
|
||
"56.8650 60.6200 Пионерский"
|
||
)
|
||
|
||
build_payload() {
|
||
local lat="$1" lon="$2" src="$3" mode="$4"
|
||
local sources_json
|
||
case "$src" in
|
||
yandex) sources_json='["yandex"]' ;;
|
||
avito) sources_json='["avito"]' ;;
|
||
cian) sources_json='["cian"]' ;;
|
||
all) sources_json='["avito","cian","yandex"]' ;;
|
||
*) sources_json='["avito","cian","yandex"]' ;;
|
||
esac
|
||
if [[ "$mode" == "deep" ]]; then
|
||
echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"deep_yandex\":true,\"multi_room_cian\":true}"
|
||
else
|
||
echo "{\"lat\":$lat,\"lon\":$lon,\"radius_m\":1500,\"sources\":$sources_json,\"multi_room_yandex\":true,\"multi_room_cian\":true}"
|
||
fi
|
||
}
|
||
|
||
log "=== cron-scrape start: source=$SOURCE mode=$MODE ==="
|
||
|
||
# Circuit breaker: если Avito IP-rate-limited (3 нуля подряд) — пропускаем оставшиеся.
|
||
zero_fetch_streak=0
|
||
for anchor in "${ANCHORS[@]}"; do
|
||
if [[ "$zero_fetch_streak" -ge 3 ]]; then
|
||
log "⚠ circuit-breaker tripped (3× zero-fetch in a row) — skipping remaining anchors"
|
||
break
|
||
fi
|
||
read -r lat lon name <<< "$anchor"
|
||
payload=$(build_payload "$lat" "$lon" "$SOURCE" "$MODE")
|
||
log "→ $name ($lat, $lon) payload=$payload"
|
||
resp=$(curl -s -X POST "$API/api/v1/admin/scrape" \
|
||
-H "Content-Type: application/json" \
|
||
-d "$payload" \
|
||
-m 300 || echo '{"error":"curl_timeout"}')
|
||
log "← $name $resp"
|
||
# Парсим total_fetched — если 0, инкремент streak; иначе reset
|
||
fetched=$(echo "$resp" | grep -oE '"total_fetched":[0-9]+' | grep -oE '[0-9]+' || echo 0)
|
||
if [[ "${fetched:-0}" == "0" ]]; then
|
||
zero_fetch_streak=$((zero_fetch_streak + 1))
|
||
else
|
||
zero_fetch_streak=0
|
||
fi
|
||
sleep 5
|
||
done
|
||
|
||
log "=== cron-scrape done ==="
|