NASP/rpki
84
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260609 增加三RP持续对比监控

Browse Source
This commit is contained in:
yuyr
2026-06-09 17:44:30 +08:00
parent d0224d53f6
commit 74012c686d
22 changed files with 3911 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

265
scripts/inter_rp/control_remote200_rp.sh Executable file
View File

@@ -0,0 +1,265 @@
#!/usr/bin/env bash
set -euo pipefail
REMOTE_HOST="${REMOTE_HOST:-root@43.110.128.200}"
REMOTE_CONFIG="${REMOTE_CONFIG:-/root/inter-rp-runners/inter-rp.env}"
REMOTE_SCRIPTS_DIR="${REMOTE_SCRIPTS_DIR:-/root/inter-rp-runners/scripts}"
DEFAULT_RETAIN_RUNS="${RETAIN_RUNS:-20}"
DEFAULT_RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
usage() {
cat <<'USAGE'
Usage:
control_remote200_rp.sh status [rpki-client|routinator|all]
control_remote200_rp.sh start <rpki-client|routinator>
control_remote200_rp.sh stop <rpki-client|routinator|all>
control_remote200_rp.sh restart <rpki-client|routinator>
Environment overrides:
REMOTE_HOST=root@43.110.128.200
REMOTE_CONFIG=/root/inter-rp-runners/inter-rp.env
REMOTE_SCRIPTS_DIR=/root/inter-rp-runners/scripts
RETAIN_RUNS=20
RSS_SAMPLE_MS=500
Notes:
- start uses the remote run_single_rp_with_rss.sh loop and runs until stopped.
- stop only kills the selected RP loop branch and its current child processes.
- rpki-client and routinator are managed independently.
USAGE
}
ACTION="${1:-status}"
RP="${2:-all}"
case "$ACTION" in
status|start|stop|restart) ;;
-h|--help|help)
usage
exit 0
;;
*)
echo "unknown action: $ACTION" >&2
usage >&2
exit 2
;;
esac
case "$RP" in
rpki-client|routinator|all) ;;
*)
echo "unknown RP: $RP" >&2
usage >&2
exit 2
;;
esac
if [[ "$ACTION" == "start" || "$ACTION" == "restart" ]] && [[ "$RP" == "all" ]]; then
echo "start/restart requires one RP: rpki-client or routinator" >&2
exit 2
fi
ssh "$REMOTE_HOST" \
"REMOTE_CONFIG='$REMOTE_CONFIG' REMOTE_SCRIPTS_DIR='$REMOTE_SCRIPTS_DIR' RETAIN_RUNS='$DEFAULT_RETAIN_RUNS' RSS_SAMPLE_MS='$DEFAULT_RSS_SAMPLE_MS' ACTION='$ACTION' RP='$RP' bash -s" <<'REMOTE'
set -euo pipefail
load_config() {
if [[ -f "$REMOTE_CONFIG" ]]; then
# shellcheck disable=SC1090
source "$REMOTE_CONFIG"
fi
INTER_RP_ROOT="${INTER_RP_ROOT:-/var/lib/inter-rp-runners}"
RETAIN_RUNS="${RETAIN_RUNS:-20}"
RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
ROUTINATOR_RUN_COMMAND="${ROUTINATOR_RUN_COMMAND:-/root/inter-rp-runners/scripts/run_routinator_once.sh}"
RPKI_CLIENT_RUN_COMMAND="${RPKI_CLIENT_RUN_COMMAND:-/root/inter-rp-runners/scripts/run_rpki_client_official_98_once.sh}"
}
rp_root_name() {
case "$1" in
routinator) echo "routinator" ;;
rpki-client) echo "rpki-client" ;;
esac
}
rp_pid_file() {
echo "$INTER_RP_ROOT/$(rp_root_name "$1").loop.pid"
}
rp_log_file() {
echo "$INTER_RP_ROOT/$(rp_root_name "$1").loop.log"
}
rp_command() {
case "$1" in
routinator) echo "$ROUTINATOR_RUN_COMMAND" ;;
rpki-client) echo "$RPKI_CLIENT_RUN_COMMAND" ;;
esac
}
rp_binary_pattern() {
case "$1" in
routinator) echo "/root/inter-rp-runners/bin/routinator|[[:space:]]routinator[[:space:]]" ;;
rpki-client) echo "rpki-client-official-9\\.8|[[:space:]]rpki-client[[:space:]]" ;;
esac
}
loop_pattern() {
local rp="$1"
echo "run_single_rp_with_rss\\.sh --rp $rp "
}
is_running_pid() {
local pid="${1:-}"
[[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null
}
current_loop_pids() {
local rp="$1"
pgrep -af "$(loop_pattern "$rp")" 2>/dev/null | awk '{print $1}' || true
}
status_one() {
local rp="$1"
local pid_file
pid_file="$(rp_pid_file "$rp")"
local pid=""
[[ -f "$pid_file" ]] && pid="$(cat "$pid_file" 2>/dev/null || true)"
echo "== $rp =="
if is_running_pid "$pid"; then
echo "loop: running pid=$pid"
else
local detected
detected="$(current_loop_pids "$rp" | paste -sd, -)"
if [[ -n "$detected" ]]; then
echo "loop: running detected_pids=$detected"
else
echo "loop: stopped"
fi
fi
ps -eo pid,ppid,stat,etime,%cpu,%mem,rss,cmd --sort=-%cpu \
| grep -E "$(loop_pattern "$rp")|$(rp_binary_pattern "$rp")" \
| grep -v grep \
| head -20 || true
local latest="$INTER_RP_ROOT/$(rp_root_name "$rp")/latest/run-meta.json"
if [[ -f "$latest" ]]; then
python3 - "$latest" <<'PY'
import json, sys
path = sys.argv[1]
with open(path, "r", encoding="utf-8") as handle:
meta = json.load(handle)
print("latest:", "runSeq=", meta.get("runSeq"), "success=", meta.get("success"), "wallMs=", meta.get("wallMs"), "vrps=", meta.get("counts", {}).get("vrps"), "vaps=", meta.get("counts", {}).get("vaps"), "rssKb=", meta.get("maxRssKb", {}).get("aggregatePeak"))
PY
else
echo "latest: none"
fi
}
start_one() {
local rp="$1"
local pid_file log_file root_name run_command
pid_file="$(rp_pid_file "$rp")"
log_file="$(rp_log_file "$rp")"
root_name="$(rp_root_name "$rp")"
run_command="$(rp_command "$rp")"
mkdir -p "$INTER_RP_ROOT/$root_name" "$INTER_RP_ROOT"
local existing=""
if [[ -f "$pid_file" ]]; then
existing="$(cat "$pid_file" 2>/dev/null || true)"
fi
if is_running_pid "$existing"; then
echo "$rp already running pid=$existing"
return 0
fi
local detected
detected="$(current_loop_pids "$rp" | head -1)"
if [[ -n "$detected" ]] && is_running_pid "$detected"; then
echo "$detected" >"$pid_file"
echo "$rp already running detected pid=$detected"
return 0
fi
nohup bash -lc '
set -euo pipefail
while true; do
"'"$REMOTE_SCRIPTS_DIR"'/run_single_rp_with_rss.sh" \
--rp "'"$rp"'" \
--root "'"$INTER_RP_ROOT/$root_name"'" \
--command "'"$run_command"'" \
--retain-runs "'"$RETAIN_RUNS"'" \
--sample-ms "'"$RSS_SAMPLE_MS"'" || true
done
' >"$log_file" 2>&1 &
local pid="$!"
echo "$pid" >"$pid_file"
echo "$rp started pid=$pid log=$log_file"
}
stop_one() {
local rp="$1"
local candidates=()
local pid_file pid
pid_file="$(rp_pid_file "$rp")"
if [[ -f "$pid_file" ]]; then
pid="$(cat "$pid_file" 2>/dev/null || true)"
[[ -n "$pid" ]] && candidates+=("$pid")
fi
while read -r pid; do
[[ -n "$pid" ]] && candidates+=("$pid")
done < <(current_loop_pids "$rp")
while read -r pid; do
[[ -n "$pid" ]] && candidates+=("$pid")
done < <(pgrep -af "$(rp_binary_pattern "$rp")" 2>/dev/null | awk '{print $1}' || true)
if ((${#candidates[@]} == 0)); then
echo "$rp already stopped"
rm -f "$pid_file"
return 0
fi
local unique_pids
unique_pids="$(printf '%s\n' "${candidates[@]}" | awk '!seen[$0]++' | tr '\n' ' ')"
echo "stopping $rp pids=$unique_pids"
for pid in $unique_pids; do
kill -TERM "$pid" 2>/dev/null || true
done
sleep 3
for pid in $unique_pids; do
kill -KILL "$pid" 2>/dev/null || true
done
rm -f "$pid_file"
echo "$rp stopped"
}
load_config
case "$ACTION:$RP" in
status:all)
date -Is
uptime
status_one routinator
status_one rpki-client
;;
status:*)
date -Is
uptime
status_one "$RP"
;;
start:*)
start_one "$RP"
status_one "$RP"
;;
stop:all)
stop_one routinator
stop_one rpki-client
;;
stop:*)
stop_one "$RP"
status_one "$RP"
;;
restart:*)
stop_one "$RP"
start_one "$RP"
status_one "$RP"
;;
esac
REMOTE

26
scripts/inter_rp/inter-rp.env.example Normal file
View File

@@ -0,0 +1,26 @@
# 远端200 runner 配置示例。
# 该脚本只负责任务调度、RSS采样、run目录和run-meta.json；具体RP命令由下面两个变量提供。
# 远端200上的运行根目录。
# 建议把运行态数据放到 /var/lib，避免 rpki-client 降权后无法读写 /root。
INTER_RP_ROOT=/var/lib/inter-rp-runners
# -1 表示持续运行；正整数表示每个RP运行多少轮。
MAX_RUNS=-1
# 每轮结束后的等待秒数。
RUN_INTERVAL_SECS=0
# 每个RP保留最近多少轮。
RETAIN_RUNS=20
# RSS采样间隔，毫秒。
RSS_SAMPLE_MS=500
# Routinator 命令模板。命令运行时会自动导出 RUN_DIR/RP_ROOT/RUN_SEQ/RUN_ID/RP_NAME。
# 命令必须把 vrps.csv/vaps.csv 写入 $RUN_DIR；Routinator 使用 --enable-aspa + JSON 输出转换出 VAP CSV。
ROUTINATOR_RUN_COMMAND=/root/inter-rp-runners/scripts/run_routinator_once.sh
# 官方 rpki-client 9.8 命令模板。
# rpki-client 会降权运行，cache 和 output 必须预先放在可写目录；命令需复制 output/csv、output/rpki.ccr，并从 output/json 转换出 vaps.csv。
RPKI_CLIENT_RUN_COMMAND=/root/inter-rp-runners/scripts/run_rpki_client_official_98_once.sh

25
scripts/inter_rp/run_inter_rp_metrics_sidecar.sh Executable file
View File

@@ -0,0 +1,25 @@
#!/usr/bin/env bash
set -euo pipefail
CONFIG_FILE="${INTER_RP_METRICS_CONFIG:-./inter-rp-metrics.env}"
if [[ -f "$CONFIG_FILE" ]]; then
# shellcheck disable=SC1090
source "$CONFIG_FILE"
fi
RPKI_INTER_RP_METRICS_BIN="${RPKI_INTER_RP_METRICS_BIN:-./bin/rpki_inter_rp_metrics}"
OURS_RUN_ROOT="${OURS_RUN_ROOT:?OURS_RUN_ROOT is required}"
PEER_ROOT="${PEER_ROOT:-/root/inter-rp-aggregator/synced-from-200}"
LISTEN="${INTER_RP_METRICS_LISTEN:-0.0.0.0:9557}"
POLL_SECS="${INTER_RP_METRICS_POLL_SECS:-30}"
INSTANCE="${INTER_RP_METRICS_INSTANCE:-remote231-inter-rp}"
LOG_DIR="${INTER_RP_METRICS_LOG_DIR:-./logs}"
mkdir -p "$LOG_DIR"
exec "$RPKI_INTER_RP_METRICS_BIN" \
--ours-run-root "$OURS_RUN_ROOT" \
--peer-root "$PEER_ROOT" \
--listen "$LISTEN" \
--poll-secs "$POLL_SECS" \
--instance "$INSTANCE" \
>>"$LOG_DIR/inter-rp-metrics.log" 2>&1

62
scripts/inter_rp/run_remote200_rp_loops.sh Executable file
View File

@@ -0,0 +1,62 @@
#!/usr/bin/env bash
set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
CONFIG_FILE="${INTER_RP_CONFIG:-./inter-rp.env}"
if [[ -f "$CONFIG_FILE" ]]; then
# shellcheck disable=SC1090
source "$CONFIG_FILE"
fi
INTER_RP_ROOT="${INTER_RP_ROOT:-/root/inter-rp-runners}"
MAX_RUNS="${MAX_RUNS:--1}"
RUN_INTERVAL_SECS="${RUN_INTERVAL_SECS:-0}"
RETAIN_RUNS="${RETAIN_RUNS:-20}"
RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
ROUTINATOR_RUN_COMMAND="${ROUTINATOR_RUN_COMMAND:-}"
RPKI_CLIENT_RUN_COMMAND="${RPKI_CLIENT_RUN_COMMAND:-}"
if [[ -z "$ROUTINATOR_RUN_COMMAND" || -z "$RPKI_CLIENT_RUN_COMMAND" ]]; then
cat >&2 <<'MSG'
ROUTINATOR_RUN_COMMAND and RPKI_CLIENT_RUN_COMMAND are required.
Each command runs with RUN_DIR, RP_ROOT, RUN_SEQ, RUN_ID and RP_NAME exported.
MSG
exit 2
fi
mkdir -p "$INTER_RP_ROOT"
run_loop() {
local rp_name="$1"
local run_command="$2"
local rp_root="$INTER_RP_ROOT/$rp_name"
local completed="0"
mkdir -p "$rp_root"
while true; do
if [[ "$MAX_RUNS" =~ ^[0-9]+$ ]] && (( completed >= MAX_RUNS )); then
break
fi
"$SCRIPT_DIR/run_single_rp_with_rss.sh" \
--rp "$rp_name" \
--root "$rp_root" \
--command "$run_command" \
--retain-runs "$RETAIN_RUNS" \
--sample-ms "$RSS_SAMPLE_MS" || true
completed=$((completed + 1))
if (( RUN_INTERVAL_SECS > 0 )); then
sleep "$RUN_INTERVAL_SECS"
fi
done
}
run_loop routinator "$ROUTINATOR_RUN_COMMAND" >"$INTER_RP_ROOT/routinator.loop.log" 2>&1 &
ROUTINATOR_LOOP_PID=$!
run_loop rpki-client "$RPKI_CLIENT_RUN_COMMAND" >"$INTER_RP_ROOT/rpki-client.loop.log" 2>&1 &
RPKI_CLIENT_LOOP_PID=$!
cat >"$INTER_RP_ROOT/runner-pids.env" <<PIDS
ROUTINATOR_LOOP_PID=$ROUTINATOR_LOOP_PID
RPKI_CLIENT_LOOP_PID=$RPKI_CLIENT_LOOP_PID
PIDS
wait "$ROUTINATOR_LOOP_PID" "$RPKI_CLIENT_LOOP_PID"

62
scripts/inter_rp/run_routinator_once.sh Executable file
View File

@@ -0,0 +1,62 @@
#!/usr/bin/env bash
set -euo pipefail
: "${RUN_DIR:?RUN_DIR is required}"
: "${RP_ROOT:?RP_ROOT is required}"
timeout 1800 /root/inter-rp-runners/bin/routinator \
--repository-dir "$RP_ROOT/repository" \
--no-rir-tals \
--extra-tals-dir /var/lib/inter-rp-runners/fixtures/tal \
--enable-aspa \
vrps --format json --output "$RUN_DIR/routinator-output.json"
python3 - "$RUN_DIR/routinator-output.json" "$RUN_DIR/vrps.csv" "$RUN_DIR/vaps.csv" <<'PY'
import csv
import json
import sys
json_path, vrps_path, vaps_path = sys.argv[1:4]
with open(json_path, "r", encoding="utf-8") as handle:
payload = json.load(handle)
def asn_text(value):
if isinstance(value, int):
return f"AS{value}"
text = str(value)
return text if text.startswith("AS") else f"AS{text}"
def asn_sort_key(value):
text = asn_text(value)
try:
return int(text[2:])
except ValueError:
return text
with open(vrps_path, "w", encoding="utf-8", newline="") as handle:
writer = csv.writer(handle)
writer.writerow(["ASN", "IP Prefix", "Max Length", "Trust Anchor"])
for row in payload.get("roas", []):
writer.writerow([
asn_text(row.get("asn", "")),
row.get("prefix", ""),
row.get("maxLength", row.get("max_length", "")),
row.get("ta", row.get("trust_anchor", "")),
])
with open(vaps_path, "w", encoding="utf-8", newline="") as handle:
writer = csv.writer(handle)
writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
for row in payload.get("aspas", payload.get("aspa", [])):
providers = sorted({asn_text(provider) for provider in row.get("providers", [])}, key=asn_sort_key)
writer.writerow([
asn_text(row.get("customer", row.get("customer_asid", ""))),
";".join(providers),
row.get("ta", row.get("trust_anchor", "unknown")),
])
PY
if [[ "${KEEP_INTER_RP_RAW_JSON:-0}" != "1" ]]; then
rm -f "$RUN_DIR/routinator-output.json"
fi

59
scripts/inter_rp/run_rpki_client_official_98_once.sh Executable file
View File

@@ -0,0 +1,59 @@
#!/usr/bin/env bash
set -euo pipefail
: "${RUN_DIR:?RUN_DIR is required}"
: "${RP_ROOT:?RP_ROOT is required}"
install -d -o nobody -g nogroup "$RP_ROOT/cache" "$RUN_DIR/output"
chown nobody:nogroup "$RUN_DIR"
timeout 1800 /root/inter-rp-runners/bin/rpki-client-official-9.8 \
-c \
-j \
-t /var/lib/inter-rp-runners/fixtures/tal/afrinic.tal \
-t /var/lib/inter-rp-runners/fixtures/tal/apnic-rfc7730-https.tal \
-t /var/lib/inter-rp-runners/fixtures/tal/arin.tal \
-t /var/lib/inter-rp-runners/fixtures/tal/lacnic.tal \
-t /var/lib/inter-rp-runners/fixtures/tal/ripe-ncc.tal \
-d "$RP_ROOT/cache" \
"$RUN_DIR/output"
cp "$RUN_DIR/output/csv" "$RUN_DIR/vrps.csv"
cp "$RUN_DIR/output/rpki.ccr" "$RUN_DIR/result.ccr"
python3 - "$RUN_DIR/output/json" "$RUN_DIR/vaps.csv" <<'PY'
import csv
import json
import sys
json_path, vaps_path = sys.argv[1:3]
with open(json_path, "r", encoding="utf-8") as handle:
payload = json.load(handle)
def asn_text(value):
if isinstance(value, int):
return f"AS{value}"
text = str(value)
return text if text.startswith("AS") else f"AS{text}"
def asn_sort_key(value):
text = asn_text(value)
try:
return int(text[2:])
except ValueError:
return text
with open(vaps_path, "w", encoding="utf-8", newline="") as handle:
writer = csv.writer(handle)
writer.writerow(["Customer ASN", "Providers", "Trust Anchor"])
for row in payload.get("aspas", []):
providers = sorted({asn_text(provider) for provider in row.get("providers", [])}, key=asn_sort_key)
writer.writerow([
asn_text(row.get("customer_asid", row.get("customer", ""))),
";".join(providers),
row.get("ta", row.get("trust_anchor", "unknown")),
])
PY
rm -rf "$RUN_DIR/output"

266
scripts/inter_rp/run_single_rp_with_rss.sh Executable file
View File

@@ -0,0 +1,266 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
cat <<'USAGE'
Usage: run_single_rp_with_rss.sh --rp <name> --root <path> --command <shell-command> [--retain-runs <n>] [--sample-ms <n>]
The command runs with RUN_DIR, RP_ROOT, RUN_SEQ, RUN_ID, and RP_NAME exported.
It must write artifacts into RUN_DIR. The wrapper writes run-meta.json and atomically updates latest.
USAGE
}
RP_NAME=""
RP_ROOT=""
RUN_COMMAND=""
RETAIN_RUNS="${RETAIN_RUNS:-20}"
SAMPLE_MS="${RSS_SAMPLE_MS:-500}"
while [[ $# -gt 0 ]]; do
case "$1" in
--rp)
RP_NAME="$2"
shift 2
;;
--root)
RP_ROOT="$2"
shift 2
;;
--command)
RUN_COMMAND="$2"
shift 2
;;
--retain-runs)
RETAIN_RUNS="$2"
shift 2
;;
--sample-ms)
SAMPLE_MS="$2"
shift 2
;;
-h|--help)
usage
exit 0
;;
*)
echo "unknown argument: $1" >&2
usage >&2
exit 2
;;
esac
done
if [[ -z "$RP_NAME" || -z "$RP_ROOT" || -z "$RUN_COMMAND" ]]; then
usage >&2
exit 2
fi
mkdir -p "$RP_ROOT/runs" "$RP_ROOT/logs"
next_seq() {
local max_seq="0"
local run_path
shopt -s nullglob
for run_path in "$RP_ROOT"/runs/run_*; do
local run_name="${run_path##*/}"
local seq="${run_name#run_}"
if [[ "$seq" =~ ^[0-9]+$ ]] && (( 10#$seq > max_seq )); then
max_seq=$((10#$seq))
fi
done
shopt -u nullglob
printf '%06d' $((max_seq + 1))
}
rss_kb_for_pid() {
local pid="$1"
awk '/^VmRSS:/ {print $2; found=1} END {if (!found) print 0}' "/proc/$pid/status" 2>/dev/null || echo 0
}
collect_related_pids() {
local root_pid="$1"
local process_group="$2"
local queue=("$root_pid")
local pid
declare -A seen=()
if [[ -n "$process_group" ]]; then
while read -r pid; do
[[ -n "$pid" ]] && queue+=("$pid")
done < <(pgrep -g "$process_group" 2>/dev/null || true)
fi
while ((${#queue[@]} > 0)); do
pid="${queue[0]}"
queue=("${queue[@]:1}")
[[ -z "$pid" || -n "${seen[$pid]:-}" ]] && continue
seen[$pid]=1
echo "$pid"
while read -r child_pid; do
[[ -n "$child_pid" ]] && queue+=("$child_pid")
done < <(pgrep -P "$pid" 2>/dev/null || true)
done
}
sum_related_rss() {
local process_group="$1"
local parent_pid="$2"
local total_rss="0"
local child_max_rss="0"
local pid
while read -r pid; do
[[ -z "$pid" ]] && continue
local rss
rss="$(rss_kb_for_pid "$pid")"
total_rss=$((total_rss + rss))
if [[ "$pid" != "$parent_pid" ]] && (( rss > child_max_rss )); then
child_max_rss="$rss"
fi
done < <(collect_related_pids "$parent_pid" "$process_group")
printf '%s %s
' "$total_rss" "$child_max_rss"
}
count_csv_rows() {
local path="$1"
if [[ ! -f "$path" ]]; then
echo 0
return
fi
awk 'BEGIN {count=0} /^[[:space:]]*$/ {next} /^#/ {next} NR==1 {next} {count++} END {print count}' "$path"
}
RUN_SEQ="$(next_seq)"
RUN_ID="run_${RUN_SEQ}"
RUN_DIR="$RP_ROOT/runs/$RUN_ID"
mkdir -p "$RUN_DIR"
STARTED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
START_EPOCH_MS="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
export RP_NAME RP_ROOT RUN_SEQ RUN_ID RUN_DIR RUN_COMMAND
set +e
setsid bash -lc "$RUN_COMMAND" >"$RUN_DIR/stdout.log" 2>"$RUN_DIR/stderr.log" &
CHILD_PID=$!
set -e
sleep 0.05
PROCESS_GROUP="$(ps -o pgid= -p "$CHILD_PID" 2>/dev/null | tr -d '[:space:]' || true)"
if [[ -z "$PROCESS_GROUP" ]]; then
PROCESS_GROUP="$CHILD_PID"
fi
PARENT_MAX_RSS_KB="0"
CHILD_MAX_RSS_KB="0"
AGGREGATE_PEAK_RSS_KB="0"
SAMPLE_INTERVAL_SECONDS="$(python3 - <<PY
print(max(float($SAMPLE_MS) / 1000.0, 0.05))
PY
)"
while kill -0 "$CHILD_PID" 2>/dev/null; do
parent_rss="$(rss_kb_for_pid "$CHILD_PID")"
read -r aggregate_rss child_rss < <(sum_related_rss "$PROCESS_GROUP" "$CHILD_PID")
if (( parent_rss > PARENT_MAX_RSS_KB )); then
PARENT_MAX_RSS_KB="$parent_rss"
fi
if (( child_rss > CHILD_MAX_RSS_KB )); then
CHILD_MAX_RSS_KB="$child_rss"
fi
if (( aggregate_rss > AGGREGATE_PEAK_RSS_KB )); then
AGGREGATE_PEAK_RSS_KB="$aggregate_rss"
fi
sleep "$SAMPLE_INTERVAL_SECONDS"
done
set +e
wait "$CHILD_PID"
EXIT_CODE=$?
set -e
FINISHED_AT="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
END_EPOCH_MS="$(python3 - <<'PY'
import time
print(int(time.time() * 1000))
PY
)"
WALL_MS=$((END_EPOCH_MS - START_EPOCH_MS))
SUCCESS=false
if [[ "$EXIT_CODE" == "0" ]]; then
SUCCESS=true
fi
VRPS_COUNT="$(count_csv_rows "$RUN_DIR/vrps.csv")"
VAPS_COUNT="$(count_csv_rows "$RUN_DIR/vaps.csv")"
export STARTED_AT FINISHED_AT WALL_MS EXIT_CODE SUCCESS
export PARENT_MAX_RSS_KB CHILD_MAX_RSS_KB AGGREGATE_PEAK_RSS_KB SAMPLE_MS
export VRPS_COUNT VAPS_COUNT
export CCR_ARTIFACT_PATH=""
export VRPS_ARTIFACT_PATH=""
export VAPS_ARTIFACT_PATH=""
if [[ -f "$RUN_DIR/result.ccr" ]]; then
CCR_ARTIFACT_PATH="result.ccr"
fi
if [[ -f "$RUN_DIR/vrps.csv" ]]; then
VRPS_ARTIFACT_PATH="vrps.csv"
fi
if [[ -f "$RUN_DIR/vaps.csv" ]]; then
VAPS_ARTIFACT_PATH="vaps.csv"
fi
python3 - <<'PY' >"$RUN_DIR/run-meta.json"
import os
import json, socket
def optional(name):
value = os.environ.get(name, "")
return value if value else None
meta = {
"schemaVersion": 1,
"rp": os.environ["RP_NAME"],
"runSeq": int(os.environ["RUN_SEQ"]),
"runId": os.environ["RUN_ID"],
"host": socket.gethostname(),
"command": os.environ["RUN_COMMAND"],
"startedAtRfc3339Utc": os.environ["STARTED_AT"],
"finishedAtRfc3339Utc": os.environ["FINISHED_AT"],
"wallMs": int(os.environ["WALL_MS"]),
"exitCode": int(os.environ["EXIT_CODE"]),
"success": os.environ["SUCCESS"] == "true",
"maxRssKb": {
"parent": int(os.environ["PARENT_MAX_RSS_KB"]),
"childMax": int(os.environ["CHILD_MAX_RSS_KB"]),
"aggregatePeak": int(os.environ["AGGREGATE_PEAK_RSS_KB"]),
"sampleIntervalMs": int(os.environ["SAMPLE_MS"]),
},
"artifacts": {
"vrpsCsv": optional("VRPS_ARTIFACT_PATH"),
"vapsCsv": optional("VAPS_ARTIFACT_PATH"),
"ccr": optional("CCR_ARTIFACT_PATH"),
"stdout": "stdout.log",
"stderr": "stderr.log",
},
"counts": {
"vrps": int(os.environ["VRPS_COUNT"]),
"vaps": int(os.environ["VAPS_COUNT"]),
},
}
print(json.dumps(meta, indent=2, ensure_ascii=False))
PY
ln -sfn "runs/$RUN_ID" "$RP_ROOT/latest.tmp"
mv -Tf "$RP_ROOT/latest.tmp" "$RP_ROOT/latest"
echo "$RUN_ID $RP_NAME exit=$EXIT_CODE wall_ms=$WALL_MS vrps=$VRPS_COUNT vaps=$VAPS_COUNT rss_kb=$AGGREGATE_PEAK_RSS_KB"
if [[ "$RETAIN_RUNS" =~ ^[0-9]+$ ]] && (( RETAIN_RUNS > 0 )); then
mapfile -t old_runs < <(find "$RP_ROOT/runs" -maxdepth 1 -type d -name 'run_*' -printf '%f\n' | sort | head -n -"$RETAIN_RUNS" || true)
for old_run in "${old_runs[@]}"; do
rm -rf "$RP_ROOT/runs/$old_run"
done
fi
exit "$EXIT_CODE"

90
scripts/inter_rp/sync_remote200_to_231.sh Executable file
View File

@@ -0,0 +1,90 @@
#!/usr/bin/env bash
set -euo pipefail
CONFIG_FILE="${INTER_RP_SYNC_CONFIG:-./inter-rp-sync.env}"
if [[ -f "$CONFIG_FILE" ]]; then
# shellcheck disable=SC1090
source "$CONFIG_FILE"
fi
REMOTE200="${REMOTE200:-root@43.110.128.200}"
REMOTE200_ROOT="${REMOTE200_ROOT:-/root/inter-rp-runners}"
PEER_ROOT="${PEER_ROOT:-/root/inter-rp-aggregator/synced-from-200}"
SYNC_INTERVAL_SECS="${SYNC_INTERVAL_SECS:-60}"
MAX_SYNCS="${MAX_SYNCS:--1}"
RPS="${RPS:-routinator rpki-client}"
mkdir -p "$PEER_ROOT"
write_status() {
local success="$1"
local message="$2"
export SYNC_SUCCESS="$success"
export SYNC_MESSAGE="$message"
export SYNC_REMOTE200="$REMOTE200"
export SYNC_REMOTE200_ROOT="$REMOTE200_ROOT"
python3 - <<'PY' >"$PEER_ROOT/sync-status.json"
import json, socket, datetime
print(json.dumps({
"schemaVersion": 1,
"success": __import__("os").environ["SYNC_SUCCESS"] == "true",
"lastSyncAtRfc3339Utc": datetime.datetime.now(datetime.UTC).replace(microsecond=0).isoformat().replace('+00:00', 'Z'),
"remoteHost": __import__("os").environ["SYNC_REMOTE200"],
"remoteRoot": __import__("os").environ["SYNC_REMOTE200_ROOT"],
"localHost": socket.gethostname(),
"message": __import__("os").environ["SYNC_MESSAGE"],
}, indent=2))
PY
}
sync_once() {
local rp_name
local temp_root="$PEER_ROOT/.sync-tmp-$$"
rm -rf "$temp_root"
mkdir -p "$temp_root"
for rp_name in $RPS; do
mkdir -p "$temp_root/$rp_name"
if ! rsync -aL --delete \
--include='run-meta.json' \
--include='result.ccr' \
--include='vrps.csv' \
--include='vaps.csv' \
--include='stdout.log' \
--include='stderr.log' \
--exclude='*' \
"$REMOTE200:$REMOTE200_ROOT/$rp_name/latest/" "$temp_root/$rp_name/latest/"; then
echo "sync failed for $rp_name; clearing stale local latest" >&2
rm -rf "$PEER_ROOT/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest.prev"
return 1
fi
if [[ ! -d "$temp_root/$rp_name/latest" ]]; then
echo "missing synced latest directory for $rp_name" >&2
rm -rf "$PEER_ROOT/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest.prev"
return 1
fi
rm -rf "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest.prev"
mkdir -p "$PEER_ROOT/$rp_name"
mv "$temp_root/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.next"
if [[ -e "$PEER_ROOT/$rp_name/latest" ]]; then
mv "$PEER_ROOT/$rp_name/latest" "$PEER_ROOT/$rp_name/latest.prev"
fi
mv "$PEER_ROOT/$rp_name/latest.next" "$PEER_ROOT/$rp_name/latest"
rm -rf "$PEER_ROOT/$rp_name/latest.prev"
done
rm -rf "$temp_root"
}
completed="0"
while true; do
if sync_once; then
write_status true "ok"
else
rm -rf "$PEER_ROOT/.sync-tmp-$$"
write_status false "rsync failed"
fi
completed=$((completed + 1))
if [[ "$MAX_SYNCS" =~ ^[0-9]+$ ]] && (( completed >= MAX_SYNCS )); then
break
fi
sleep "$SYNC_INTERVAL_SECS"
done