80 lines
2.9 KiB
Bash
Executable File
80 lines
2.9 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
REMOTE231="${REMOTE231:-root@47.251.127.231}"
|
|
REMOTE_ROOT="${REMOTE_ROOT:-/var/lib/inter-rp-runners}"
|
|
PEER_ROOT="${PEER_ROOT:-/root/ours-rp-continuous/portable-soak/inter-rp-peers}"
|
|
SYNC_INTERVAL_SECS="${SYNC_INTERVAL_SECS:-60}"
|
|
MAX_SYNCS="${MAX_SYNCS:--1}"
|
|
LOG_PREFIX="[inter-rp-sync]"
|
|
mkdir -p "$PEER_ROOT/routinator"
|
|
write_status() {
|
|
local success="$1"
|
|
local message="$2"
|
|
env SYNC_SUCCESS="$success" SYNC_MESSAGE="$message" SYNC_REMOTE="$REMOTE231" SYNC_REMOTE_ROOT="$REMOTE_ROOT" python3 - "$PEER_ROOT/sync-status.json" <<'PY'
|
|
import datetime, json, os, socket, sys
|
|
path = sys.argv[1]
|
|
payload = {
|
|
"schemaVersion": 1,
|
|
"success": os.environ["SYNC_SUCCESS"] == "true",
|
|
"lastSyncAtRfc3339Utc": datetime.datetime.now(datetime.UTC).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
|
|
"remoteHost": os.environ["SYNC_REMOTE"],
|
|
"remoteRoot": os.environ["SYNC_REMOTE_ROOT"],
|
|
"localHost": socket.gethostname(),
|
|
"message": os.environ["SYNC_MESSAGE"],
|
|
}
|
|
with open(path, "w", encoding="utf-8") as handle:
|
|
json.dump(payload, handle, indent=2)
|
|
handle.write("\n")
|
|
PY
|
|
}
|
|
sync_once() {
|
|
local tmp="$PEER_ROOT/.sync-routinator-$$"
|
|
rm -rf "$tmp"
|
|
mkdir -p "$tmp"
|
|
if ! rsync -aL --delete \
|
|
--include='run-meta.json' \
|
|
--include='result.ccr' \
|
|
--include='vrps.csv' \
|
|
--include='vaps.csv' \
|
|
--include='stdout.log' \
|
|
--include='stderr.log' \
|
|
--exclude='*' \
|
|
"$REMOTE231:$REMOTE_ROOT/routinator/latest/" "$tmp/latest/"; then
|
|
rm -rf "$tmp"
|
|
return 1
|
|
fi
|
|
if [[ ! -f "$tmp/latest/run-meta.json" ]]; then
|
|
rm -rf "$tmp"
|
|
return 2
|
|
fi
|
|
if ! ssh -o BatchMode=yes -o ConnectTimeout=8 "$REMOTE231" "curl -fsS --max-time 20 http://127.0.0.1:9558/metrics" >"$tmp/routinator-metrics.prom"; then
|
|
rm -rf "$tmp"
|
|
return 3
|
|
fi
|
|
rm -rf "$PEER_ROOT/routinator/latest.next" "$PEER_ROOT/routinator/latest.prev" "$PEER_ROOT/routinator/routinator-metrics.prom.next"
|
|
mv "$tmp/routinator-metrics.prom" "$PEER_ROOT/routinator/routinator-metrics.prom.next"
|
|
mv "$PEER_ROOT/routinator/routinator-metrics.prom.next" "$PEER_ROOT/routinator/routinator-metrics.prom"
|
|
mv "$tmp/latest" "$PEER_ROOT/routinator/latest.next"
|
|
if [[ -e "$PEER_ROOT/routinator/latest" ]]; then
|
|
mv "$PEER_ROOT/routinator/latest" "$PEER_ROOT/routinator/latest.prev"
|
|
fi
|
|
mv "$PEER_ROOT/routinator/latest.next" "$PEER_ROOT/routinator/latest"
|
|
rm -rf "$PEER_ROOT/routinator/latest.prev" "$tmp"
|
|
}
|
|
completed=0
|
|
while true; do
|
|
if sync_once; then
|
|
echo "$LOG_PREFIX $(date -u +%Y-%m-%dT%H:%M:%SZ) ok"
|
|
write_status true "ok"
|
|
else
|
|
code=$?
|
|
echo "$LOG_PREFIX $(date -u +%Y-%m-%dT%H:%M:%SZ) sync failed code=$code" >&2
|
|
write_status false "routinator rsync failed code=$code"
|
|
fi
|
|
completed=$((completed + 1))
|
|
if [[ "$MAX_SYNCS" =~ ^[0-9]+$ ]] && (( completed >= MAX_SYNCS )); then
|
|
break
|
|
fi
|
|
sleep "$SYNC_INTERVAL_SECS"
|
|
done
|