20260624 修复future notBefore PP缓存并固化231发布

2026-06-24 13:36:39 +08:00
parent adb68fd469
commit af25316d68
6 changed files with 1064 additions and 47 deletions
--- a/scripts/inter_rp/run_inter_rp_metrics_sidecar.sh
+++ b/scripts/inter_rp/run_inter_rp_metrics_sidecar.sh
@@ -7,19 +7,41 @@ if [[ -f "$CONFIG_FILE" ]]; then
  source "$CONFIG_FILE"
 fi

+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+INTER_RP_METRICS_EXPORTER="${INTER_RP_METRICS_EXPORTER:-ours-routinator}"
 RPKI_INTER_RP_METRICS_BIN="${RPKI_INTER_RP_METRICS_BIN:-./bin/rpki_inter_rp_metrics}"
+OURS_ROUTINATOR_EXPORTER="${OURS_ROUTINATOR_EXPORTER:-$SCRIPT_DIR/inter_rp_ours_routinator_exporter.py}"
 OURS_RUN_ROOT="${OURS_RUN_ROOT:?OURS_RUN_ROOT is required}"
 PEER_ROOT="${PEER_ROOT:-/root/inter-rp-aggregator/synced-from-200}"
 LISTEN="${INTER_RP_METRICS_LISTEN:-0.0.0.0:9557}"
 POLL_SECS="${INTER_RP_METRICS_POLL_SECS:-30}"
+SCAN_TTL_SECONDS="${INTER_RP_SCAN_TTL_SECONDS:-20}"
 INSTANCE="${INTER_RP_METRICS_INSTANCE:-remote231-inter-rp}"
 LOG_DIR="${INTER_RP_METRICS_LOG_DIR:-./logs}"

 mkdir -p "$LOG_DIR"
-exec "$RPKI_INTER_RP_METRICS_BIN" \
-  --ours-run-root "$OURS_RUN_ROOT" \
-  --peer-root "$PEER_ROOT" \
-  --listen "$LISTEN" \
-  --poll-secs "$POLL_SECS" \
-  --instance "$INSTANCE" \
-  >>"$LOG_DIR/inter-rp-metrics.log" 2>&1
+case "$INTER_RP_METRICS_EXPORTER" in
+  ours-routinator)
+    exec env \
+      OURS_RUN_ROOT="$OURS_RUN_ROOT" \
+      PEER_ROOT="$PEER_ROOT" \
+      INTER_RP_INSTANCE="$INSTANCE" \
+      INTER_RP_LISTEN="$LISTEN" \
+      INTER_RP_SCAN_TTL_SECONDS="$SCAN_TTL_SECONDS" \
+      "$OURS_ROUTINATOR_EXPORTER" \
+      >>"$LOG_DIR/inter-rp-metrics.log" 2>&1
+    ;;
+  rust-generic)
+    exec "$RPKI_INTER_RP_METRICS_BIN" \
+      --ours-run-root "$OURS_RUN_ROOT" \
+      --peer-root "$PEER_ROOT" \
+      --listen "$LISTEN" \
+      --poll-secs "$POLL_SECS" \
+      --instance "$INSTANCE" \
+      >>"$LOG_DIR/inter-rp-metrics.log" 2>&1
+    ;;
+  *)
+    echo "unknown INTER_RP_METRICS_EXPORTER: $INTER_RP_METRICS_EXPORTER" >&2
+    exit 2
+    ;;
+esac
--- a/scripts/soak/build_portable_soak_package.sh
+++ b/scripts/soak/build_portable_soak_package.sh
@@ -89,6 +89,8 @@ install -m 0755 "$SCRIPT_DIR/run_soak.sh" "$STAGE_DIR/run_soak.sh"
 install -m 0755 "$SCRIPT_DIR/run_24h_soak_with_metrics.sh" "$STAGE_DIR/run_24h_soak_with_metrics.sh"
 install -m 0755 "$SCRIPT_DIR/fixed_phase_loop.sh" "$STAGE_DIR/scripts/soak/fixed_phase_loop.sh"
 install -m 0755 "$SCRIPT_DIR/hourly_soak_report.py" "$STAGE_DIR/scripts/soak/hourly_soak_report.py"
+install -m 0755 "$SCRIPT_DIR/publish_remote231.sh" "$STAGE_DIR/scripts/soak/publish_remote231.sh"
+install -m 0755 "$SCRIPT_DIR/publish_remote231_full.sh" "$STAGE_DIR/scripts/soak/publish_remote231_full.sh"
 install -m 0644 "$SCRIPT_DIR/portable-soak.env.example" "$STAGE_DIR/.env"
 install -m 0644 "$SCRIPT_DIR/portable-soak.env.example" "$STAGE_DIR/portable-soak.env.example"

@@ -115,6 +117,7 @@ cp -a "$REPO_ROOT/tests/fixtures/tal" "$STAGE_DIR/fixtures/"
 cp -a "$REPO_ROOT/tests/fixtures/ta" "$STAGE_DIR/fixtures/"
 cp -a "$REPO_ROOT/scripts/periodic" "$STAGE_DIR/scripts/"
 cp -a "$REPO_ROOT/scripts/cir" "$STAGE_DIR/scripts/"
+cp -a "$REPO_ROOT/scripts/inter_rp" "$STAGE_DIR/scripts/"
 cp -a "$REPO_ROOT/monitor" "$STAGE_DIR/"
 find "$STAGE_DIR/scripts" -type d -name __pycache__ -prune -exec rm -rf {} +

--- a/scripts/soak/publish_remote231_full.sh
+++ b/scripts/soak/publish_remote231_full.sh
@@ -0,0 +1,630 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+
+REMOTE_HOST="${REMOTE_HOST:-root@47.251.127.231}"
+REMOTE_ROOT="${REMOTE_ROOT:-/root/ours-rp-continuous/portable-soak}"
+MODE="${MODE:-dry-run}"
+PUBLISH_MODE="${PUBLISH_MODE:-snapshot}"
+PROFILE="${PROFILE:-release}"
+OUT_DIR="${OUT_DIR:-$REPO_ROOT/target/remote231_publish}"
+PACKAGE_PREFIX="${PACKAGE_PREFIX:-remote231-publish}"
+PACKAGE_ARCHIVE="${PACKAGE_ARCHIVE:-}"
+BUILD_PACKAGE="${BUILD_PACKAGE:-1}"
+RESTART_QUERY_SERVICE="${RESTART_QUERY_SERVICE:-1}"
+START_ARTIFACT_METRICS="${START_ARTIFACT_METRICS:-1}"
+START_QUERY_SERVICE="${START_QUERY_SERVICE:-1}"
+START_INTER_RP="${START_INTER_RP:-1}"
+START_MONITOR_STACK="${START_MONITOR_STACK:-1}"
+START_RPKI_SOAK="${START_RPKI_SOAK:-1}"
+RESTART_FIXED_PHASE_LOOP="${RESTART_FIXED_PHASE_LOOP:-1}"
+START_ROUTINATOR_SYNC="${START_ROUTINATOR_SYNC:-1}"
+VERIFY_INTER_RP_DASHBOARD="${VERIFY_INTER_RP_DASHBOARD:-1}"
+WAIT_FIRST_RUN="${WAIT_FIRST_RUN:-1}"
+FIRST_RUN_TIMEOUT_SECS="${FIRST_RUN_TIMEOUT_SECS:-7200}"
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  scripts/soak/publish_remote231_full.sh [--snapshot|--delta] [--execute|--dry-run]
+                                        [--package <portable-soak.tar.gz>]
+                                        [--remote-host <root@host>] [--remote-root <path>]
+
+Builds or reuses a portable soak package, publishes it to remote231, restores the fixed-phase
+ours RP soak loop, and starts/verifies sidecars:
+  - rpki_artifact_metrics (:9556)
+  - rpki_query_service (:9560, optional)
+  - ours-rp vs Routinator inter-RP exporter (:9557)
+  - local Routinator artifact sync helper
+  - Prometheus/Grafana monitor stack
+
+Modes:
+  --snapshot  Preserve run history but move state/db aside; first new run is snapshot.
+  --delta     Preserve state/db and continue from current DB; next run should be delta.
+
+Default is dry-run + snapshot.
+
+Environment overrides:
+  REMOTE_HOST=root@47.251.127.231
+  REMOTE_ROOT=/root/ours-rp-continuous/portable-soak
+  PROFILE=release
+  BUILD_PACKAGE=1
+  RESTART_QUERY_SERVICE=1
+  START_QUERY_SERVICE=1
+  START_INTER_RP=1
+  START_MONITOR_STACK=1
+  WAIT_FIRST_RUN=1
+USAGE
+}
+
+die() {
+  echo "error: $*" >&2
+  exit 2
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --snapshot)
+      PUBLISH_MODE="snapshot"
+      ;;
+    --delta)
+      PUBLISH_MODE="delta"
+      ;;
+    --execute)
+      MODE="execute"
+      ;;
+    --dry-run)
+      MODE="dry-run"
+      ;;
+    --package)
+      shift
+      PACKAGE_ARCHIVE="${1:?--package requires a value}"
+      BUILD_PACKAGE=0
+      ;;
+    --remote-host)
+      shift
+      REMOTE_HOST="${1:?--remote-host requires a value}"
+      ;;
+    --remote-root)
+      shift
+      REMOTE_ROOT="${1:?--remote-root requires a value}"
+      ;;
+    --profile)
+      shift
+      PROFILE="${1:?--profile requires a value}"
+      ;;
+    --no-query-service)
+      START_QUERY_SERVICE=0
+      RESTART_QUERY_SERVICE=0
+      ;;
+    --no-monitor-stack)
+      START_MONITOR_STACK=0
+      ;;
+    --no-inter-rp)
+      START_INTER_RP=0
+      ;;
+    --no-wait-first-run)
+      WAIT_FIRST_RUN=0
+      ;;
+    --help|-h)
+      usage
+      exit 0
+      ;;
+    *)
+      die "unknown argument: $1"
+      ;;
+  esac
+  shift
+done
+
+case "$MODE" in
+  dry-run|execute) ;;
+  *) die "invalid mode: $MODE" ;;
+esac
+case "$PUBLISH_MODE" in
+  snapshot|delta) ;;
+  *) die "invalid publish mode: $PUBLISH_MODE" ;;
+esac
+
+run_or_echo() {
+  if [[ "$MODE" == "execute" ]]; then
+    "$@"
+  else
+    printf '[dry-run] '
+    printf '%q ' "$@"
+    printf '\n'
+  fi
+}
+
+bool_arg() {
+  case "${1:-0}" in
+    1|true|TRUE|yes|YES|on|ON) return 0 ;;
+    *) return 1 ;;
+  esac
+}
+
+require_command() {
+  command -v "$1" >/dev/null 2>&1 || die "missing required command: $1"
+}
+
+require_command ssh
+require_command scp
+require_command python3
+
+if bool_arg "$BUILD_PACKAGE"; then
+  if [[ "$PROFILE" == "release" ]]; then
+    TARGET_DIR="$REPO_ROOT/target/release"
+  else
+    TARGET_DIR="$REPO_ROOT/target/$PROFILE"
+  fi
+  required_bins=(rpki rpki_daemon db_stats rpki_artifact_metrics rpki_query_service rpki_query_indexer)
+  missing_bins=()
+  for bin in "${required_bins[@]}"; do
+    [[ -x "$TARGET_DIR/$bin" ]] || missing_bins+=("$bin")
+  done
+  if (( ${#missing_bins[@]} > 0 )); then
+    die "missing required $PROFILE binaries: ${missing_bins[*]}; build them before publish"
+  fi
+  PACKAGE_ARCHIVE="$(
+    OUT_DIR="$OUT_DIR" PACKAGE_PREFIX="$PACKAGE_PREFIX" \
+      "$SCRIPT_DIR/build_portable_soak_package.sh" --profile "$PROFILE"
+  )"
+else
+  [[ -n "$PACKAGE_ARCHIVE" ]] || die "--package is required when BUILD_PACKAGE=0"
+fi
+[[ -f "$PACKAGE_ARCHIVE" ]] || die "package not found: $PACKAGE_ARCHIVE"
+
+echo "remote231 full publish mode=$MODE publish_mode=$PUBLISH_MODE remote=$REMOTE_HOST root=$REMOTE_ROOT package=$PACKAGE_ARCHIVE"
+if [[ -n "$(git -C "$REPO_ROOT" status --short 2>/dev/null || true)" ]]; then
+  echo "warning: local rpki worktree is dirty; package manifest records dirty provenance" >&2
+fi
+
+if [[ "$PUBLISH_MODE" == "snapshot" ]]; then
+  ssh "$REMOTE_HOST" "bash -s -- '$REMOTE_ROOT' '$MODE'" <<'REMOTE'
+set -euo pipefail
+remote_root="$1"
+mode="$2"
+matching_pids() {
+  local pattern="$1"
+  pgrep -af "$pattern" 2>/dev/null | while IFS= read -r line; do
+    local pid="${line%% *}"
+    local cmd="${line#* }"
+    [[ "$pid" =~ ^[0-9]+$ ]] || continue
+    [[ "$pid" == "$$" || "$pid" == "$BASHPID" || "$pid" == "${PPID:-}" ]] && continue
+    [[ "$cmd" == *"bash -s --"* && "$cmd" == *"$remote_root"* ]] && continue
+    printf '%s\n' "$pid"
+  done | sort -u
+}
+if [[ "$mode" == "execute" ]]; then
+  mapfile -t pids < <(matching_pids "fixed_phase_loop.sh --name ours-rp")
+  if (( ${#pids[@]} > 0 )); then
+    kill -TERM "${pids[@]}" >/dev/null 2>&1 || true
+    sleep 2
+  fi
+else
+  echo "[dry-run] stop existing fixed_phase_loop.sh --name ours-rp before snapshot publish"
+fi
+REMOTE
+  publish_args=(
+    "$SCRIPT_DIR/publish_remote231.sh"
+    --package "$PACKAGE_ARCHIVE"
+    --remote-host "$REMOTE_HOST"
+    --remote-root "$REMOTE_ROOT"
+  )
+  if bool_arg "$RESTART_QUERY_SERVICE"; then
+    publish_args+=(--restart-query-service)
+  fi
+  if [[ "$MODE" == "execute" ]]; then
+    publish_args+=(--execute)
+  else
+    publish_args+=(--dry-run)
+  fi
+  "${publish_args[@]}"
+  if [[ "$MODE" == "execute" && "$START_QUERY_SERVICE" == "1" && "$RESTART_QUERY_SERVICE" == "1" ]]; then
+    START_QUERY_SERVICE=0
+  fi
+  WAIT_FIRST_RUN=0
+else
+  REMOTE_STAGE_PARENT="/root/rpki_publish_packages"
+  PACKAGE_BASENAME="$(basename "$PACKAGE_ARCHIVE")"
+  REMOTE_ARCHIVE="$REMOTE_STAGE_PARENT/$PACKAGE_BASENAME"
+  ssh "$REMOTE_HOST" "mkdir -p '$REMOTE_STAGE_PARENT'"
+  scp "$PACKAGE_ARCHIVE" "$REMOTE_HOST:$REMOTE_ARCHIVE"
+  ssh "$REMOTE_HOST" "bash -s -- '$REMOTE_ROOT' '$REMOTE_ARCHIVE' '$MODE'" <<'REMOTE'
+set -euo pipefail
+remote_root="$1"
+remote_archive="$2"
+mode="$3"
+
+log() { printf '[delta-publish] %s\n' "$*"; }
+run_or_echo() {
+  if [[ "$mode" == "execute" ]]; then
+    "$@"
+  else
+    printf '[dry-run] '
+    printf '%q ' "$@"
+    printf '\n'
+  fi
+}
+matching_pids() {
+  local pattern="$1"
+  pgrep -af "$pattern" 2>/dev/null | while IFS= read -r line; do
+    local pid="${line%% *}"
+    local cmd="${line#* }"
+    [[ "$pid" =~ ^[0-9]+$ ]] || continue
+    [[ "$pid" == "$$" || "$pid" == "$BASHPID" || "$pid" == "${PPID:-}" ]] && continue
+    [[ "$cmd" == *"bash -s --"* && "$cmd" == *"$remote_root"* ]] && continue
+    printf '%s\n' "$pid"
+  done | sort -u
+}
+terminate_matching() {
+  local signal="$1"
+  local pattern="$2"
+  local -a pids=()
+  mapfile -t pids < <(matching_pids "$pattern")
+  if (( ${#pids[@]} > 0 )); then
+    kill "$signal" "${pids[@]}" >/dev/null 2>&1 || true
+  fi
+}
+json_status_is_success() {
+  local path="$1"
+  python3 - "$path" <<'PY'
+import json, sys
+try:
+    data = json.load(open(sys.argv[1], encoding="utf-8"))
+except Exception:
+    sys.exit(1)
+sys.exit(0 if data.get("status") == "success" else 1)
+PY
+}
+max_successful_run_name() {
+  local candidate
+  find "$remote_root/runs" -maxdepth 1 -type d -name 'run_*' -printf '%f\n' 2>/dev/null | sort -V | while read -r candidate; do
+    [[ -n "$candidate" ]] || continue
+    if json_status_is_success "$remote_root/runs/$candidate/run-meta.json" \
+      && json_status_is_success "$remote_root/runs/$candidate/run-summary.json"; then
+      printf '%s\n' "$candidate"
+    fi
+  done | tail -1
+}
+
+[[ -d "$remote_root" ]] || { echo "remote root not found: $remote_root" >&2; exit 2; }
+[[ -f "$remote_archive" ]] || { echo "archive not found: $remote_archive" >&2; exit 2; }
+last_run="$(max_successful_run_name || true)"
+[[ -n "$last_run" ]] || { echo "no successful run found under $remote_root/runs" >&2; exit 2; }
+timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
+backup_root="$remote_root/state/backups/pre_delta_publish_${timestamp}_after_${last_run}"
+extract_root="$remote_root/state/publish-staging/$timestamp"
+new_pkg="$extract_root/portable-soak"
+log "last_successful_run=$last_run backup_root=$backup_root mode=$mode"
+
+if [[ "$mode" == "execute" ]]; then
+  terminate_matching -TERM "$remote_root/bin/rpki "
+  terminate_matching -TERM "$remote_root/bin/rpki_daemon "
+  terminate_matching -TERM "$remote_root/run_soak.sh"
+  terminate_matching -TERM "fixed_phase_loop.sh --name ours-rp"
+  sleep 3
+  terminate_matching -KILL "$remote_root/bin/rpki "
+  terminate_matching -KILL "$remote_root/bin/rpki_daemon "
+  terminate_matching -KILL "$remote_root/run_soak.sh"
+  terminate_matching -KILL "fixed_phase_loop.sh --name ours-rp"
+else
+  log "would stop current ours-rp soak/fixed-phase processes"
+fi
+
+run_or_echo mkdir -p "$backup_root" "$extract_root"
+if [[ "$mode" == "execute" ]]; then
+  tar -C "$extract_root" -xzf "$remote_archive"
+  [[ -x "$new_pkg/bin/rpki" ]] || { echo "extracted package missing bin/rpki" >&2; exit 5; }
+fi
+if [[ -f "$remote_root/.env" ]]; then
+  run_or_echo cp -a "$remote_root/.env" "$backup_root/env.before"
+fi
+if [[ -d "$remote_root/bin" ]]; then
+  run_or_echo mv "$remote_root/bin" "$backup_root/bin.before"
+fi
+for path in run_soak.sh run_24h_soak_with_metrics.sh scripts monitor fixtures copied-binaries.txt missing-optional-binaries.txt fixtures.txt scripts.txt manifest.json portable-soak.env.example; do
+  if [[ "$mode" == "execute" && ! -e "$new_pkg/$path" ]]; then
+    continue
+  fi
+  if [[ "$mode" == "execute" ]]; then
+    rm -rf "$remote_root/$path"
+    cp -a "$new_pkg/$path" "$remote_root/$path"
+  else
+    printf '[dry-run] replace %s from package\n' "$path"
+  fi
+done
+run_or_echo cp -a "$new_pkg/bin" "$remote_root/bin"
+if [[ -f "$backup_root/env.before" ]]; then
+  run_or_echo cp -a "$backup_root/env.before" "$remote_root/.env"
+fi
+run_or_echo mkdir -p "$remote_root/state/db" "$remote_root/state/meta" "$remote_root/tmp" "$remote_root/logs"
+if [[ "$mode" == "execute" ]]; then
+  chmod +x "$remote_root/run_soak.sh" "$remote_root/run_24h_soak_with_metrics.sh" "$remote_root/bin/"* "$remote_root/scripts/soak/"* "$remote_root/scripts/inter_rp/"* 2>/dev/null || true
+fi
+REMOTE
+fi
+
+ssh "$REMOTE_HOST" "bash -s -- '$REMOTE_ROOT' '$MODE' '$START_RPKI_SOAK' '$START_ARTIFACT_METRICS' '$START_QUERY_SERVICE' '$START_INTER_RP' '$START_ROUTINATOR_SYNC' '$START_MONITOR_STACK' '$VERIFY_INTER_RP_DASHBOARD' '$WAIT_FIRST_RUN' '$FIRST_RUN_TIMEOUT_SECS' '$RESTART_FIXED_PHASE_LOOP'" <<'REMOTE'
+set -euo pipefail
+remote_root="$1"
+mode="$2"
+start_rpki_soak="$3"
+start_artifact_metrics="$4"
+start_query_service="$5"
+start_inter_rp="$6"
+start_routinator_sync="$7"
+start_monitor_stack="$8"
+verify_inter_rp_dashboard="$9"
+wait_first_run="${10}"
+first_run_timeout_secs="${11}"
+restart_fixed_phase_loop="${12}"
+
+log() { printf '[remote231-full] %s\n' "$*"; }
+is_true() {
+  case "${1:-0}" in 1|true|TRUE|yes|YES|on|ON) return 0 ;; *) return 1 ;; esac
+}
+run_or_echo() {
+  if [[ "$mode" == "execute" ]]; then
+    "$@"
+  else
+    printf '[dry-run] '
+    printf '%q ' "$@"
+    printf '\n'
+  fi
+}
+matching_pids() {
+  local pattern="$1"
+  pgrep -af "$pattern" 2>/dev/null | while IFS= read -r line; do
+    local pid="${line%% *}"
+    local cmd="${line#* }"
+    [[ "$pid" =~ ^[0-9]+$ ]] || continue
+    [[ "$pid" == "$$" || "$pid" == "$BASHPID" || "$pid" == "${PPID:-}" ]] && continue
+    [[ "$cmd" == *"bash -s --"* && "$cmd" == *"$remote_root"* ]] && continue
+    printf '%s\n' "$pid"
+  done | sort -u
+}
+stop_matching() {
+  local pattern="$1"
+  if [[ "$mode" != "execute" ]]; then
+    log "would stop processes matching: $pattern"
+    return 0
+  fi
+  local -a pids=()
+  mapfile -t pids < <(matching_pids "$pattern")
+  if (( ${#pids[@]} > 0 )); then
+    kill -TERM "${pids[@]}" >/dev/null 2>&1 || true
+    sleep 1
+  fi
+}
+ensure_artifact_metrics() {
+  is_true "$start_artifact_metrics" || return 0
+  if curl -fsS --max-time 3 http://127.0.0.1:9556/healthz >/dev/null 2>&1 \
+    || curl -fsS --max-time 3 http://127.0.0.1:9556/metrics >/dev/null 2>&1; then
+    log "artifact metrics already up"
+    return 0
+  fi
+  stop_matching "$remote_root/bin/rpki_artifact_metrics"
+  if [[ "$mode" == "execute" ]]; then
+    nohup "$remote_root/bin/rpki_artifact_metrics" \
+      --run-root "$remote_root" \
+      --listen 0.0.0.0:9556 \
+      --poll-secs 120 \
+      --instance remote231-continuous \
+      >"$remote_root/logs/artifact-metrics.full-publish.log" 2>&1 &
+  else
+    log "would start artifact metrics on :9556"
+  fi
+}
+ensure_query_service() {
+  is_true "$start_query_service" || return 0
+  stop_matching "$remote_root/bin/rpki_query_service"
+  local latest_seq
+  latest_seq="$(find "$remote_root/runs" -maxdepth 1 -type d -name 'run_*' -printf '%f\n' 2>/dev/null | sort -V | tail -1 | sed 's/run_//;s/^0*//')"
+  [[ -n "$latest_seq" ]] || latest_seq=0
+  local min_seq=$((latest_seq + 1))
+  log "starting query service watch_min_run_seq=$min_seq"
+  if [[ "$mode" == "execute" ]]; then
+    nohup "$remote_root/bin/rpki_query_service" \
+      --query-db "$remote_root/state/query-db" \
+      --repo-bytes-db "$remote_root/state/db/repo-bytes.db" \
+      --export-root "$remote_root/state/query-exports" \
+      --listen 0.0.0.0:9560 \
+      --watch-run-root "$remote_root" \
+      --watch-interval-secs 60 \
+      --watch-min-run-seq "$min_seq" \
+      --retain-indexed-runs 10 \
+      --indexer-bin "$remote_root/bin/rpki_query_indexer" \
+      --projection-entry-limit 20 \
+      >"$remote_root/logs/query-service.full-publish.log" 2>&1 &
+  else
+    log "would start query service on :9560"
+  fi
+}
+ensure_inter_rp() {
+  is_true "$start_inter_rp" || return 0
+  stop_matching "rpki_inter_rp_metrics"
+  stop_matching "inter_rp_ours_routinator_exporter.py"
+  if [[ "$mode" == "execute" ]]; then
+    nohup env \
+      OURS_RUN_ROOT="$remote_root" \
+      PEER_ROOT="$remote_root/inter-rp-peers" \
+      INTER_RP_INSTANCE=remote231-inter-rp \
+      INTER_RP_LISTEN=0.0.0.0:9557 \
+      INTER_RP_SCAN_TTL_SECONDS=20 \
+      "$remote_root/scripts/inter_rp/inter_rp_ours_routinator_exporter.py" \
+      >"$remote_root/logs/inter-rp-metrics.full-publish.log" 2>&1 &
+  else
+    log "would start inter-rp ours+routinator exporter on :9557"
+  fi
+}
+ensure_routinator_sync() {
+  is_true "$start_routinator_sync" || return 0
+  if pgrep -af "sync_local_routinator_peer.sh" >/dev/null 2>&1; then
+    log "routinator local sync already running"
+    return 0
+  fi
+  if [[ -x "$remote_root/ops/sync_local_routinator_peer.sh" ]]; then
+    if [[ "$mode" == "execute" ]]; then
+      nohup "$remote_root/ops/sync_local_routinator_peer.sh" >"$remote_root/logs/sync-local-routinator.full-publish.log" 2>&1 &
+    else
+      log "would start existing ops/sync_local_routinator_peer.sh"
+    fi
+  else
+    log "warning: missing $remote_root/ops/sync_local_routinator_peer.sh; inter-RP routinator latest may stale"
+  fi
+}
+ensure_monitor_stack() {
+  is_true "$start_monitor_stack" || return 0
+  if [[ -f "$remote_root/monitor/docker-compose.yml" ]]; then
+    if command -v docker >/dev/null 2>&1; then
+      if [[ "$mode" == "execute" ]]; then
+        (cd "$remote_root/monitor" && docker compose up -d)
+      else
+        log "would run docker compose up -d under $remote_root/monitor"
+      fi
+    else
+      log "warning: docker not installed; cannot start monitor stack"
+    fi
+  else
+    log "warning: missing monitor/docker-compose.yml"
+  fi
+}
+ensure_fixed_phase_soak() {
+  is_true "$start_rpki_soak" || return 0
+  if pgrep -af "fixed_phase_loop.sh --name ours-rp" >/dev/null 2>&1; then
+    log "ours-rp fixed phase loop already running"
+    return 0
+  fi
+  is_true "$restart_fixed_phase_loop" || { log "fixed phase loop restart disabled"; return 0; }
+  if [[ "$mode" == "execute" ]]; then
+    nohup bash "$remote_root/scripts/soak/fixed_phase_loop.sh" \
+      --name ours-rp \
+      --cycle-secs 900 \
+      --offset-secs 0 \
+      --lock-file /var/lock/rpki-heavy-run.lock \
+      --lock-wait-secs 60 \
+      -- env PACKAGE_ROOT="$remote_root" ENV_FILE="$remote_root/.env" "$remote_root/run_soak.sh" \
+      >"$remote_root/logs/fixed-phase-ours.full-publish.log" 2>&1 &
+  else
+    log "would start ours-rp fixed phase loop"
+  fi
+}
+wait_url() {
+  local name="$1"
+  local url="$2"
+  local deadline=$((SECONDS + 60))
+  while (( SECONDS < deadline )); do
+    if curl -fsS --max-time 5 "$url" >/dev/null 2>&1; then
+      log "$name up"
+      return 0
+    fi
+    sleep 2
+  done
+  echo "$name did not become ready: $url" >&2
+  return 1
+}
+verify_prometheus_queries() {
+  is_true "$verify_inter_rp_dashboard" || return 0
+  [[ "$mode" == "execute" ]] || { log "would verify dashboard PromQL queries"; return 0; }
+  python3 - "$remote_root/monitor/grafana/dashboards/ours-rp-inter-rp.json" <<'PY'
+import json, pathlib, urllib.parse, urllib.request, sys
+path = pathlib.Path(sys.argv[1])
+if not path.exists():
+    print("missing inter-rp dashboard json", file=sys.stderr)
+    sys.exit(1)
+dash = json.loads(path.read_text())
+exprs = []
+def walk(value):
+    if isinstance(value, dict):
+        if "expr" in value:
+            exprs.append(value["expr"])
+        for item in value.values():
+            walk(item)
+    elif isinstance(value, list):
+        for item in value:
+            walk(item)
+walk(dash)
+empty = []
+for expr in sorted(set(exprs)):
+    url = "http://127.0.0.1:9090/api/v1/query?query=" + urllib.parse.quote(expr)
+    data = json.load(urllib.request.urlopen(url, timeout=10))
+    if not data.get("data", {}).get("result", []):
+        empty.append(expr)
+if empty:
+    print("empty dashboard queries:")
+    print("\n".join(empty))
+    sys.exit(1)
+print(f"inter-rp dashboard queries ok count={len(set(exprs))}")
+PY
+}
+wait_next_run_if_requested() {
+  is_true "$wait_first_run" || return 0
+  [[ "$mode" == "execute" ]] || { log "would wait for next run completion"; return 0; }
+  local before latest deadline status
+  before="$(find "$remote_root/runs" -maxdepth 1 -type d -name 'run_*' -printf '%f\n' 2>/dev/null | sort -V | tail -1)"
+  deadline=$((SECONDS + first_run_timeout_secs))
+  log "waiting for next completed run after ${before:-none}"
+  while (( SECONDS < deadline )); do
+    latest="$(find "$remote_root/runs" -maxdepth 1 -type d -name 'run_*' -printf '%f\n' 2>/dev/null | sort -V | tail -1)"
+    if [[ -n "$latest" && "$latest" != "$before" && -f "$remote_root/runs/$latest/run-summary.json" ]]; then
+      status="$(python3 - "$remote_root/runs/$latest/run-summary.json" <<'PY'
+import json, sys
+try:
+    print(json.load(open(sys.argv[1], encoding="utf-8")).get("status", "missing"))
+except Exception:
+    print("missing")
+PY
+)"
+      if [[ "$status" == "success" ]]; then
+        python3 - "$remote_root/runs/$latest/run-summary.json" <<'PY'
+import json, sys
+s = json.load(open(sys.argv[1], encoding="utf-8"))
+st = s.get("stageTiming") or {}
+rc = s.get("reportCounts") or {}
+print(
+    "next run success "
+    f"run={s.get('runId')} wall_ms={s.get('wallMs')} "
+    f"validation_ms={st.get('validation_ms')} repo_sync_ms_total={st.get('repo_sync_ms_total')} "
+    f"vrps={rc.get('vrps')} aspas={rc.get('aspas')} pp={rc.get('publicationPoints')}"
+)
+PY
+        return 0
+      fi
+    fi
+    sleep 5
+  done
+  echo "timeout waiting for next completed run" >&2
+  return 1
+}
+
+ensure_artifact_metrics
+ensure_query_service
+ensure_inter_rp
+ensure_routinator_sync
+ensure_monitor_stack
+ensure_fixed_phase_soak
+
+if [[ "$mode" == "execute" ]]; then
+  sleep 5
+  wait_url "artifact metrics" http://127.0.0.1:9556/metrics
+  if is_true "$start_query_service"; then wait_url "query service" http://127.0.0.1:9560/api/v1; fi
+  if is_true "$start_inter_rp"; then wait_url "inter-rp metrics" http://127.0.0.1:9557/metrics; fi
+  if is_true "$start_monitor_stack"; then wait_url "prometheus" http://127.0.0.1:9090/-/ready; wait_url "grafana" http://127.0.0.1:3000/api/health; fi
+  if is_true "$start_inter_rp"; then
+    curl -fsS http://127.0.0.1:9557/metrics | grep -E 'inter_rp_(service_last_reload_success|parse_errors|repo_sync_overlap_total|vrps_diff_by_class)' | head -30
+  fi
+  verify_prometheus_queries
+fi
+
+wait_next_run_if_requested
+
+log "process summary"
+pgrep -af 'fixed_phase_loop|run_soak.sh|rpki_artifact_metrics|rpki_query_service|inter_rp_ours_routinator_exporter|prometheus|grafana|routinator' || true
+log "df"
+df -h / /root 2>/dev/null | sort -u || true
+REMOTE
+
+echo "remote231 full publish finished mode=$MODE publish_mode=$PUBLISH_MODE"