20260626_2 优化child证书缓存并修复远端发布

scripts/soak/publish_remote231_full.sh

@@ -21,9 +21,12 @@ START_MONITOR_STACK="${START_MONITOR_STACK:-1}"
 START_RPKI_SOAK="${START_RPKI_SOAK:-1}"
 RESTART_FIXED_PHASE_LOOP="${RESTART_FIXED_PHASE_LOOP:-1}"
 START_ROUTINATOR_SYNC="${START_ROUTINATOR_SYNC:-1}"
+START_ROUTINATOR_LOCAL_SERVICES="${START_ROUTINATOR_LOCAL_SERVICES:-1}"
 VERIFY_INTER_RP_DASHBOARD="${VERIFY_INTER_RP_DASHBOARD:-1}"
 WAIT_FIRST_RUN="${WAIT_FIRST_RUN:-1}"
 FIRST_RUN_TIMEOUT_SECS="${FIRST_RUN_TIMEOUT_SECS:-7200}"
+SNAPSHOT_STOP_SIDE_SERVICES="${SNAPSHOT_STOP_SIDE_SERVICES:-1}"
+ROUTINATOR_READY_TIMEOUT_SECS="${ROUTINATOR_READY_TIMEOUT_SECS:-300}"
 
 usage() {
   cat <<'USAGE'
@@ -37,6 +40,7 @@ ours RP soak loop, and starts/verifies sidecars:
   - rpki_artifact_metrics (:9556)
   - rpki_query_service (:9560, optional)
   - ours-rp vs Routinator inter-RP exporter (:9557)
+  - local Routinator server/loop used by the inter-RP dashboard
   - local Routinator artifact sync helper
   - Prometheus/Grafana monitor stack
 
@@ -54,8 +58,11 @@ Environment overrides:
   RESTART_QUERY_SERVICE=1
   START_QUERY_SERVICE=1
   START_INTER_RP=1
+  START_ROUTINATOR_LOCAL_SERVICES=1
+  SNAPSHOT_STOP_SIDE_SERVICES=1
   START_MONITOR_STACK=1
   WAIT_FIRST_RUN=1
+  ROUTINATOR_READY_TIMEOUT_SECS=300
 USAGE
 }
 
@@ -105,6 +112,12 @@ while [[ $# -gt 0 ]]; do
     --no-inter-rp)
       START_INTER_RP=0
       ;;
+    --no-routinator-local-services)
+      START_ROUTINATOR_LOCAL_SERVICES=0
+      ;;
+    --no-snapshot-stop-side-services)
+      SNAPSHOT_STOP_SIDE_SERVICES=0
+      ;;
     --no-wait-first-run)
       WAIT_FIRST_RUN=0
       ;;
@@ -182,10 +195,14 @@ if [[ -n "$(git -C "$REPO_ROOT" status --short 2>/dev/null || true)" ]]; then
 fi
 
 if [[ "$PUBLISH_MODE" == "snapshot" ]]; then
-  ssh "$REMOTE_HOST" "bash -s -- '$REMOTE_ROOT' '$MODE'" <<'REMOTE'
+  ssh "$REMOTE_HOST" "bash -s -- '$REMOTE_ROOT' '$MODE' '$SNAPSHOT_STOP_SIDE_SERVICES'" <<'REMOTE'
 set -euo pipefail
 remote_root="$1"
 mode="$2"
+snapshot_stop_side_services="$3"
+is_true() {
+  case "${1:-0}" in 1|true|TRUE|yes|YES|on|ON) return 0 ;; *) return 1 ;; esac
+}
 matching_pids() {
   local pattern="$1"
   pgrep -af "$pattern" 2>/dev/null | while IFS= read -r line; do
@@ -197,14 +214,58 @@ matching_pids() {
     printf '%s\n' "$pid"
   done | sort -u
 }
-if [[ "$mode" == "execute" ]]; then
-  mapfile -t pids < <(matching_pids "fixed_phase_loop.sh --name ours-rp")
+terminate_matching() {
+  local signal="$1"
+  local pattern="$2"
+  local label="$3"
+  local -a pids=()
+  mapfile -t pids < <(matching_pids "$pattern")
   if (( ${#pids[@]} > 0 )); then
-    kill -TERM "${pids[@]}" >/dev/null 2>&1 || true
-    sleep 2
+    if [[ "$mode" == "execute" ]]; then
+      printf '[snapshot-prestop] %s %s pids=%s\n' "$signal" "$label" "${pids[*]}"
+      kill "$signal" "${pids[@]}" >/dev/null 2>&1 || true
+    else
+      printf '[dry-run] stop %s pids=%s pattern=%s\n' "$label" "${pids[*]}" "$pattern"
+    fi
+  else
+    printf '[snapshot-prestop] no %s processes\n' "$label"
+  fi
+}
+if [[ "$mode" == "execute" ]]; then
+  terminate_matching -TERM "fixed_phase_loop.sh --name ours-rp" "ours fixed phase loop"
+  if is_true "$snapshot_stop_side_services"; then
+    terminate_matching -TERM "$remote_root/bin/rpki_query_service" "query service"
+    terminate_matching -TERM "$remote_root/bin/rpki_artifact_metrics" "artifact metrics"
+    terminate_matching -TERM "inter_rp_ours_routinator_exporter.py" "inter-rp exporter"
+    terminate_matching -TERM "rpki_inter_rp_metrics" "inter-rp exporter"
+    terminate_matching -TERM "sync_local_routinator_peer.sh" "local routinator sync"
+    terminate_matching -TERM "fixed_phase_loop.sh --name routinator" "routinator fixed phase loop"
+    terminate_matching -TERM "/root/inter-rp-runners/scripts/run_single_rp_with_rss.sh --rp routinator" "routinator one-shot wrapper"
+    terminate_matching -TERM "/root/inter-rp-runners/bin/routinator" "routinator process"
+  fi
+  sleep 3
+  if is_true "$snapshot_stop_side_services"; then
+    terminate_matching -KILL "$remote_root/bin/rpki_query_service" "query service"
+    terminate_matching -KILL "$remote_root/bin/rpki_artifact_metrics" "artifact metrics"
+    terminate_matching -KILL "inter_rp_ours_routinator_exporter.py" "inter-rp exporter"
+    terminate_matching -KILL "rpki_inter_rp_metrics" "inter-rp exporter"
+    terminate_matching -KILL "sync_local_routinator_peer.sh" "local routinator sync"
+    terminate_matching -KILL "fixed_phase_loop.sh --name routinator" "routinator fixed phase loop"
+    terminate_matching -KILL "/root/inter-rp-runners/scripts/run_single_rp_with_rss.sh --rp routinator" "routinator one-shot wrapper"
+    terminate_matching -KILL "/root/inter-rp-runners/bin/routinator" "routinator process"
   fi
 else
   echo "[dry-run] stop existing fixed_phase_loop.sh --name ours-rp before snapshot publish"
+  if is_true "$snapshot_stop_side_services"; then
+    terminate_matching -TERM "$remote_root/bin/rpki_query_service" "query service"
+    terminate_matching -TERM "$remote_root/bin/rpki_artifact_metrics" "artifact metrics"
+    terminate_matching -TERM "inter_rp_ours_routinator_exporter.py" "inter-rp exporter"
+    terminate_matching -TERM "rpki_inter_rp_metrics" "inter-rp exporter"
+    terminate_matching -TERM "sync_local_routinator_peer.sh" "local routinator sync"
+    terminate_matching -TERM "fixed_phase_loop.sh --name routinator" "routinator fixed phase loop"
+    terminate_matching -TERM "/root/inter-rp-runners/scripts/run_single_rp_with_rss.sh --rp routinator" "routinator one-shot wrapper"
+    terminate_matching -TERM "/root/inter-rp-runners/bin/routinator" "routinator process"
+  fi
 fi
 REMOTE
   publish_args=(
@@ -347,7 +408,7 @@ fi
 REMOTE
 fi
 
-ssh "$REMOTE_HOST" "bash -s -- '$REMOTE_ROOT' '$MODE' '$START_RPKI_SOAK' '$START_ARTIFACT_METRICS' '$START_QUERY_SERVICE' '$START_INTER_RP' '$START_ROUTINATOR_SYNC' '$START_MONITOR_STACK' '$VERIFY_INTER_RP_DASHBOARD' '$WAIT_FIRST_RUN' '$FIRST_RUN_TIMEOUT_SECS' '$RESTART_FIXED_PHASE_LOOP'" <<'REMOTE'
+ssh "$REMOTE_HOST" "bash -s -- '$REMOTE_ROOT' '$MODE' '$START_RPKI_SOAK' '$START_ARTIFACT_METRICS' '$START_QUERY_SERVICE' '$START_INTER_RP' '$START_ROUTINATOR_SYNC' '$START_ROUTINATOR_LOCAL_SERVICES' '$START_MONITOR_STACK' '$VERIFY_INTER_RP_DASHBOARD' '$WAIT_FIRST_RUN' '$FIRST_RUN_TIMEOUT_SECS' '$RESTART_FIXED_PHASE_LOOP' '$ROUTINATOR_READY_TIMEOUT_SECS'" <<'REMOTE'
 set -euo pipefail
 remote_root="$1"
 mode="$2"
@@ -356,11 +417,13 @@ start_artifact_metrics="$4"
 start_query_service="$5"
 start_inter_rp="$6"
 start_routinator_sync="$7"
-start_monitor_stack="$8"
-verify_inter_rp_dashboard="$9"
-wait_first_run="${10}"
-first_run_timeout_secs="${11}"
-restart_fixed_phase_loop="${12}"
+start_routinator_local_services="$8"
+start_monitor_stack="$9"
+verify_inter_rp_dashboard="${10}"
+wait_first_run="${11}"
+first_run_timeout_secs="${12}"
+restart_fixed_phase_loop="${13}"
+routinator_ready_timeout_secs="${14}"
 
 log() { printf '[remote231-full] %s\n' "$*"; }
 is_true() {
@@ -460,6 +523,62 @@ ensure_inter_rp() {
     log "would start inter-rp ours+routinator exporter on :9557"
   fi
 }
+ensure_routinator_local_services() {
+  is_true "$start_routinator_local_services" || return 0
+  local routinator_bin="/root/inter-rp-runners/bin/routinator"
+  local routinator_root="/var/lib/inter-rp-runners"
+  if [[ -x "$routinator_bin" ]]; then
+    if pgrep -af "$routinator_bin .* server .*127.0.0.1:9558" >/dev/null 2>&1; then
+      log "local routinator server already running"
+    elif [[ "$mode" == "execute" ]]; then
+      mkdir -p "$routinator_root/routinator-server/repository"
+      nohup "$routinator_bin" \
+        --repository-dir "$routinator_root/routinator-server/repository" \
+        --no-rir-tals \
+        --extra-tals-dir "$routinator_root/fixtures/tal" \
+        --enable-aspa \
+        server \
+        --http 127.0.0.1:9558 \
+        --rtr 127.0.0.1:0 \
+        --refresh 86400 \
+        >"$remote_root/logs/routinator-server.full-publish.log" 2>&1 &
+      log "started local routinator server on 127.0.0.1:9558"
+    else
+      log "would start local routinator server on 127.0.0.1:9558"
+    fi
+  else
+    log "warning: missing $routinator_bin; cannot start local routinator server"
+  fi
+
+  if pgrep -af "fixed_phase_loop.sh --name routinator" >/dev/null 2>&1; then
+    log "routinator fixed phase loop already running"
+    return 0
+  fi
+  if [[ -x "$remote_root/scripts/soak/fixed_phase_loop.sh" \
+    && -x /root/inter-rp-runners/scripts/run_single_rp_with_rss.sh \
+    && -x /root/inter-rp-runners/scripts/run_routinator_once.sh ]]; then
+    if [[ "$mode" == "execute" ]]; then
+      nohup bash "$remote_root/scripts/soak/fixed_phase_loop.sh" \
+        --name routinator \
+        --cycle-secs 900 \
+        --offset-secs 450 \
+        --lock-file /var/lock/rpki-heavy-run.lock \
+        --lock-wait-secs 60 \
+        -- /root/inter-rp-runners/scripts/run_single_rp_with_rss.sh \
+          --rp routinator \
+          --root /var/lib/inter-rp-runners/routinator \
+          --command /root/inter-rp-runners/scripts/run_routinator_once.sh \
+          --retain-runs 30 \
+          --sample-ms 500 \
+        >"$remote_root/logs/fixed-phase-routinator.full-publish.log" 2>&1 &
+      log "started routinator fixed phase loop"
+    else
+      log "would start routinator fixed phase loop"
+    fi
+  else
+    log "warning: missing routinator loop scripts; cannot start routinator fixed phase loop"
+  fi
+}
 ensure_routinator_sync() {
   is_true "$start_routinator_sync" || return 0
   if pgrep -af "sync_local_routinator_peer.sh" >/dev/null 2>&1; then
@@ -515,7 +634,8 @@ ensure_fixed_phase_soak() {
 wait_url() {
   local name="$1"
   local url="$2"
-  local deadline=$((SECONDS + 60))
+  local timeout_secs="${3:-60}"
+  local deadline=$((SECONDS + timeout_secs))
   while (( SECONDS < deadline )); do
     if curl -fsS --max-time 5 "$url" >/dev/null 2>&1; then
       log "$name up"
@@ -603,6 +723,7 @@ PY
 ensure_artifact_metrics
 ensure_query_service
 ensure_inter_rp
+ensure_routinator_local_services
 ensure_routinator_sync
 ensure_monitor_stack
 ensure_fixed_phase_soak
@@ -611,6 +732,7 @@ if [[ "$mode" == "execute" ]]; then
   sleep 5
   wait_url "artifact metrics" http://127.0.0.1:9556/metrics
   if is_true "$start_query_service"; then wait_url "query service" http://127.0.0.1:9560/api/v1; fi
+  if is_true "$start_routinator_local_services"; then wait_url "routinator server" http://127.0.0.1:9558/metrics "$routinator_ready_timeout_secs"; fi
   if is_true "$start_inter_rp"; then wait_url "inter-rp metrics" http://127.0.0.1:9557/metrics; fi
   if is_true "$start_monitor_stack"; then wait_url "prometheus" http://127.0.0.1:9090/-/ready; wait_url "grafana" http://127.0.0.1:3000/api/health; fi
   if is_true "$start_inter_rp"; then