#!/usr/bin/env bash set -euo pipefail REMOTE_HOST="${REMOTE_HOST:-root@43.110.128.200}" REMOTE_CONFIG="${REMOTE_CONFIG:-/root/inter-rp-runners/inter-rp.env}" REMOTE_SCRIPTS_DIR="${REMOTE_SCRIPTS_DIR:-/root/inter-rp-runners/scripts}" DEFAULT_RETAIN_RUNS="${RETAIN_RUNS:-20}" DEFAULT_RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}" usage() { cat <<'USAGE' Usage: control_remote200_rp.sh status [rpki-client|routinator|all] control_remote200_rp.sh start control_remote200_rp.sh stop control_remote200_rp.sh restart Environment overrides: REMOTE_HOST=root@43.110.128.200 REMOTE_CONFIG=/root/inter-rp-runners/inter-rp.env REMOTE_SCRIPTS_DIR=/root/inter-rp-runners/scripts RETAIN_RUNS=20 RSS_SAMPLE_MS=500 Notes: - start uses the remote run_single_rp_with_rss.sh loop and runs until stopped. - stop only kills the selected RP loop branch and its current child processes. - rpki-client and routinator are managed independently. USAGE } ACTION="${1:-status}" RP="${2:-all}" case "$ACTION" in status|start|stop|restart) ;; -h|--help|help) usage exit 0 ;; *) echo "unknown action: $ACTION" >&2 usage >&2 exit 2 ;; esac case "$RP" in rpki-client|routinator|all) ;; *) echo "unknown RP: $RP" >&2 usage >&2 exit 2 ;; esac if [[ "$ACTION" == "start" || "$ACTION" == "restart" ]] && [[ "$RP" == "all" ]]; then echo "start/restart requires one RP: rpki-client or routinator" >&2 exit 2 fi ssh "$REMOTE_HOST" \ "REMOTE_CONFIG='$REMOTE_CONFIG' REMOTE_SCRIPTS_DIR='$REMOTE_SCRIPTS_DIR' RETAIN_RUNS='$DEFAULT_RETAIN_RUNS' RSS_SAMPLE_MS='$DEFAULT_RSS_SAMPLE_MS' ACTION='$ACTION' RP='$RP' bash -s" <<'REMOTE' set -euo pipefail load_config() { if [[ -f "$REMOTE_CONFIG" ]]; then # shellcheck disable=SC1090 source "$REMOTE_CONFIG" fi INTER_RP_ROOT="${INTER_RP_ROOT:-/var/lib/inter-rp-runners}" RETAIN_RUNS="${RETAIN_RUNS:-20}" RSS_SAMPLE_MS="${RSS_SAMPLE_MS:-500}" ROUTINATOR_RUN_COMMAND="${ROUTINATOR_RUN_COMMAND:-/root/inter-rp-runners/scripts/run_routinator_once.sh}" RPKI_CLIENT_RUN_COMMAND="${RPKI_CLIENT_RUN_COMMAND:-/root/inter-rp-runners/scripts/run_rpki_client_official_98_once.sh}" } rp_root_name() { case "$1" in routinator) echo "routinator" ;; rpki-client) echo "rpki-client" ;; esac } rp_pid_file() { echo "$INTER_RP_ROOT/$(rp_root_name "$1").loop.pid" } rp_log_file() { echo "$INTER_RP_ROOT/$(rp_root_name "$1").loop.log" } rp_command() { case "$1" in routinator) echo "$ROUTINATOR_RUN_COMMAND" ;; rpki-client) echo "$RPKI_CLIENT_RUN_COMMAND" ;; esac } rp_binary_pattern() { case "$1" in routinator) echo "/root/inter-rp-runners/bin/routinator|[[:space:]]routinator[[:space:]]" ;; rpki-client) echo "rpki-client-official-9\\.8|[[:space:]]rpki-client[[:space:]]" ;; esac } loop_pattern() { local rp="$1" echo "run_single_rp_with_rss\\.sh --rp $rp " } is_running_pid() { local pid="${1:-}" [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null } current_loop_pids() { local rp="$1" pgrep -af "$(loop_pattern "$rp")" 2>/dev/null | awk '{print $1}' || true } status_one() { local rp="$1" local pid_file pid_file="$(rp_pid_file "$rp")" local pid="" [[ -f "$pid_file" ]] && pid="$(cat "$pid_file" 2>/dev/null || true)" echo "== $rp ==" if is_running_pid "$pid"; then echo "loop: running pid=$pid" else local detected detected="$(current_loop_pids "$rp" | paste -sd, -)" if [[ -n "$detected" ]]; then echo "loop: running detected_pids=$detected" else echo "loop: stopped" fi fi ps -eo pid,ppid,stat,etime,%cpu,%mem,rss,cmd --sort=-%cpu \ | grep -E "$(loop_pattern "$rp")|$(rp_binary_pattern "$rp")" \ | grep -v grep \ | head -20 || true local latest="$INTER_RP_ROOT/$(rp_root_name "$rp")/latest/run-meta.json" if [[ -f "$latest" ]]; then python3 - "$latest" <<'PY' import json, sys path = sys.argv[1] with open(path, "r", encoding="utf-8") as handle: meta = json.load(handle) print("latest:", "runSeq=", meta.get("runSeq"), "success=", meta.get("success"), "wallMs=", meta.get("wallMs"), "vrps=", meta.get("counts", {}).get("vrps"), "vaps=", meta.get("counts", {}).get("vaps"), "rssKb=", meta.get("maxRssKb", {}).get("aggregatePeak")) PY else echo "latest: none" fi } start_one() { local rp="$1" local pid_file log_file root_name run_command pid_file="$(rp_pid_file "$rp")" log_file="$(rp_log_file "$rp")" root_name="$(rp_root_name "$rp")" run_command="$(rp_command "$rp")" mkdir -p "$INTER_RP_ROOT/$root_name" "$INTER_RP_ROOT" local existing="" if [[ -f "$pid_file" ]]; then existing="$(cat "$pid_file" 2>/dev/null || true)" fi if is_running_pid "$existing"; then echo "$rp already running pid=$existing" return 0 fi local detected detected="$(current_loop_pids "$rp" | head -1)" if [[ -n "$detected" ]] && is_running_pid "$detected"; then echo "$detected" >"$pid_file" echo "$rp already running detected pid=$detected" return 0 fi nohup bash -lc ' set -euo pipefail while true; do "'"$REMOTE_SCRIPTS_DIR"'/run_single_rp_with_rss.sh" \ --rp "'"$rp"'" \ --root "'"$INTER_RP_ROOT/$root_name"'" \ --command "'"$run_command"'" \ --retain-runs "'"$RETAIN_RUNS"'" \ --sample-ms "'"$RSS_SAMPLE_MS"'" || true done ' >"$log_file" 2>&1 & local pid="$!" echo "$pid" >"$pid_file" echo "$rp started pid=$pid log=$log_file" } stop_one() { local rp="$1" local candidates=() local pid_file pid pid_file="$(rp_pid_file "$rp")" if [[ -f "$pid_file" ]]; then pid="$(cat "$pid_file" 2>/dev/null || true)" [[ -n "$pid" ]] && candidates+=("$pid") fi while read -r pid; do [[ -n "$pid" ]] && candidates+=("$pid") done < <(current_loop_pids "$rp") while read -r pid; do [[ -n "$pid" ]] && candidates+=("$pid") done < <(pgrep -af "$(rp_binary_pattern "$rp")" 2>/dev/null | awk '{print $1}' || true) if ((${#candidates[@]} == 0)); then echo "$rp already stopped" rm -f "$pid_file" return 0 fi local unique_pids unique_pids="$(printf '%s\n' "${candidates[@]}" | awk '!seen[$0]++' | tr '\n' ' ')" echo "stopping $rp pids=$unique_pids" for pid in $unique_pids; do kill -TERM "$pid" 2>/dev/null || true done sleep 3 for pid in $unique_pids; do kill -KILL "$pid" 2>/dev/null || true done rm -f "$pid_file" echo "$rp stopped" } load_config case "$ACTION:$RP" in status:all) date -Is uptime status_one routinator status_one rpki-client ;; status:*) date -Is uptime status_one "$RP" ;; start:*) start_one "$RP" status_one "$RP" ;; stop:all) stop_one routinator stop_one rpki-client ;; stop:*) stop_one "$RP" status_one "$RP" ;; restart:*) stop_one "$RP" start_one "$RP" status_one "$RP" ;; esac REMOTE