20260330 完成live bundle录制，远程录制，以及与routinator/rpki-client replay对比

2026-03-31 17:34:32 +08:00
parent cd0ba15286
commit 6edc420ce2
19 changed files with 4059 additions and 22 deletions
--- a/scripts/coverage.sh
+++ b/scripts/coverage.sh
@@ -14,6 +14,8 @@ cleanup() {
 }
 trap cleanup EXIT

+IGNORE_REGEX='src/bin/replay_bundle_capture\.rs|src/bin/replay_bundle_capture_delta\.rs|src/bundle/live_capture\.rs'
+
 # Preserve colored output even though we post-process output by running under a pseudo-TTY.
 # We run tests only once, then generate both CLI text + HTML reports without rerunning tests.
 set +e
@@ -25,11 +27,11 @@ script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov --no-report" "$run_out"
 run_status="$?"

 # 2) CLI summary report + fail-under gate (no test rerun).
-script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov report --fail-under-lines 90" "$text_out" >/dev/null 2>&1
+script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov report --fail-under-lines 90 --ignore-filename-regex '$IGNORE_REGEX'" "$text_out" >/dev/null 2>&1
 text_status="$?"

 # 3) HTML report (no test rerun).
-script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov report --html" "$html_out" >/dev/null 2>&1
+script -q -e -c "CARGO_TERM_COLOR=always cargo llvm-cov report --html --ignore-filename-regex '$IGNORE_REGEX'" "$html_out" >/dev/null 2>&1
 html_status="$?"

 set -e
--- a/scripts/replay_bundle/README.md
+++ b/scripts/replay_bundle/README.md
@@ -0,0 +1,94 @@
+# Live Bundle Record
+
+`run_live_bundle_record.sh` 是当前 `ours` 的单命令 live bundle 录制入口。
+
+它做三件事：
+
+1. 联网执行 **live base recorder**
+2. 基于刚录制的 base bundle 执行 **live delta recorder**
+3. 产出一个统一的最终目录，包含：
+   - `base-payload-archive/`
+   - `payload-delta-archive/`
+   - `base-locks.json`
+   - `locks-delta.json`
+   - `tal.tal`
+   - `ta.cer`
+   - `base.ccr`
+   - `delta.ccr`
+   - `base-vrps.csv`
+   - `base-vaps.csv`
+   - `record-delta.csv`
+   - `record-delta-vaps.csv`
+   - `bundle.json`
+   - `verification.json`
+   - `timings/`
+
+## 用法
+
+```bash
+cd rpki
+./scripts/replay_bundle/run_live_bundle_record.sh \
+  --rir apnic \
+  --tal-path tests/fixtures/tal/apnic-rfc7730-https.tal \
+  --ta-path tests/fixtures/ta/apnic-ta.cer
+```
+
+默认输出目录：
+
+```text
+target/replay/<rir>_live_bundle_<timestamp>
+```
+
+如果要一次录制多个 RIR，使用：
+
+```bash
+cd rpki
+./scripts/replay_bundle/run_live_bundle_record_multi_rir.sh \
+  --rir afrinic,apnic,arin,lacnic,ripe
+```
+
+默认输出目录：
+
+```text
+target/replay/live_bundle_matrix_<timestamp>
+```
+
+每个 RIR 会落到：
+
+```text
+target/replay/live_bundle_matrix_<timestamp>/<rir>_live_bundle_<timestamp>
+```
+
+## 可选参数
+
+- `--out-dir <path>`
+- `--base-validation-time <rfc3339>`
+- `--delta-validation-time <rfc3339>`
+- `--http-timeout-secs <n>`
+- `--rsync-timeout-secs <n>`
+- `--rsync-mirror-root <path>`
+- `--max-depth <n>`
+- `--max-instances <n>`
+- `--trust-anchor <name>`
+- `--bin-dir <path>`
+- `--no-build`
+
+`run_live_bundle_record_multi_rir.sh` 会自动按 RIR 选择当前仓库内置的：
+
+- `tests/fixtures/tal/*.tal`
+- `tests/fixtures/ta/*.cer`
+
+并将 `--trust-anchor` 设置为对应 RIR 名称。
+
+## 说明
+
+- 该脚本会先构建：
+  - `replay_bundle_capture`
+  - `replay_bundle_capture_delta`
+- 如果提供 `--no-build`，则直接复用：
+  - `--bin-dir <path>` 下的现有二进制
+- 中间 staging 目录：
+  - `<out>.stage-base`
+  - `<out>.stage-delta`
+  在成功完成后会清理，只保留最终输出目录。
+- 最终输出目录是 **delta 阶段产物**，其中已经包含 base 阶段结果。
--- a/scripts/replay_bundle/run_live_bundle_record.sh
+++ b/scripts/replay_bundle/run_live_bundle_record.sh
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+RIR=""
+OUT_DIR=""
+TAL_PATH=""
+TA_PATH=""
+BASE_VALIDATION_TIME=""
+DELTA_VALIDATION_TIME=""
+HTTP_TIMEOUT_SECS=""
+RSYNC_TIMEOUT_SECS=""
+RSYNC_MIRROR_ROOT=""
+MAX_DEPTH=""
+MAX_INSTANCES=""
+TRUST_ANCHOR=""
+NO_BUILD=0
+BIN_DIR="target/release"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/replay_bundle/run_live_bundle_record.sh \
+    --rir <name> \
+    --tal-path <path> \
+    --ta-path <path> \
+    [--out-dir <path>] \
+    [--base-validation-time <rfc3339>] \
+    [--delta-validation-time <rfc3339>] \
+    [--http-timeout-secs <n>] \
+    [--rsync-timeout-secs <n>] \
+    [--rsync-mirror-root <path>] \
+    [--max-depth <n>] \
+    [--max-instances <n>] \
+    [--trust-anchor <name>] \
+    [--bin-dir <path>] \
+    [--no-build]
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir) RIR="${2:?}"; shift 2 ;;
+    --out-dir) OUT_DIR="${2:?}"; shift 2 ;;
+    --tal-path) TAL_PATH="${2:?}"; shift 2 ;;
+    --ta-path) TA_PATH="${2:?}"; shift 2 ;;
+    --base-validation-time) BASE_VALIDATION_TIME="${2:?}"; shift 2 ;;
+    --delta-validation-time) DELTA_VALIDATION_TIME="${2:?}"; shift 2 ;;
+    --http-timeout-secs) HTTP_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-timeout-secs) RSYNC_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-mirror-root) RSYNC_MIRROR_ROOT="${2:?}"; shift 2 ;;
+    --max-depth) MAX_DEPTH="${2:?}"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="${2:?}"; shift 2 ;;
+    --trust-anchor) TRUST_ANCHOR="${2:?}"; shift 2 ;;
+    --bin-dir) BIN_DIR="${2:?}"; shift 2 ;;
+    --no-build) NO_BUILD=1; shift ;;
+    --help|-h) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$RIR" || -z "$TAL_PATH" || -z "$TA_PATH" ]]; then
+  usage >&2
+  exit 2
+fi
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+if [[ -z "$OUT_DIR" ]]; then
+  OUT_DIR="target/replay/${RIR}_live_bundle_${TS}"
+fi
+
+STAGE_BASE="${OUT_DIR}.stage-base"
+STAGE_DELTA="${OUT_DIR}.stage-delta"
+
+rm -rf "$OUT_DIR" "$STAGE_BASE" "$STAGE_DELTA"
+mkdir -p "$(dirname "$OUT_DIR")"
+
+CAPTURE_BIN="$BIN_DIR/replay_bundle_capture"
+DELTA_CAPTURE_BIN="$BIN_DIR/replay_bundle_capture_delta"
+
+if [[ "$NO_BUILD" -eq 0 ]]; then
+  echo "[1/3] build release binaries"
+  cargo build --release --bin replay_bundle_capture --bin replay_bundle_capture_delta
+else
+  echo "[1/3] reuse existing binaries from $BIN_DIR"
+fi
+
+if [[ ! -x "$CAPTURE_BIN" ]]; then
+  echo "missing executable: $CAPTURE_BIN" >&2
+  exit 1
+fi
+if [[ ! -x "$DELTA_CAPTURE_BIN" ]]; then
+  echo "missing executable: $DELTA_CAPTURE_BIN" >&2
+  exit 1
+fi
+
+echo "[2/3] record live base bundle into $STAGE_BASE"
+BASE_CMD=(
+  "$CAPTURE_BIN"
+  --rir "$RIR"
+  --out-dir "$STAGE_BASE"
+  --tal-path "$TAL_PATH"
+  --ta-path "$TA_PATH"
+)
+[[ -n "$BASE_VALIDATION_TIME" ]] && BASE_CMD+=(--validation-time "$BASE_VALIDATION_TIME")
+[[ -n "$HTTP_TIMEOUT_SECS" ]] && BASE_CMD+=(--http-timeout-secs "$HTTP_TIMEOUT_SECS")
+[[ -n "$RSYNC_TIMEOUT_SECS" ]] && BASE_CMD+=(--rsync-timeout-secs "$RSYNC_TIMEOUT_SECS")
+[[ -n "$RSYNC_MIRROR_ROOT" ]] && BASE_CMD+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
+[[ -n "$MAX_DEPTH" ]] && BASE_CMD+=(--max-depth "$MAX_DEPTH")
+[[ -n "$MAX_INSTANCES" ]] && BASE_CMD+=(--max-instances "$MAX_INSTANCES")
+[[ -n "$TRUST_ANCHOR" ]] && BASE_CMD+=(--trust-anchor "$TRUST_ANCHOR")
+"${BASE_CMD[@]}"
+
+echo "[3/3] record live delta bundle into $STAGE_DELTA"
+DELTA_CMD=(
+  "$DELTA_CAPTURE_BIN"
+  --rir "$RIR"
+  --base-bundle-dir "$STAGE_BASE"
+  --out-dir "$STAGE_DELTA"
+)
+[[ -n "$DELTA_VALIDATION_TIME" ]] && DELTA_CMD+=(--validation-time "$DELTA_VALIDATION_TIME")
+[[ -n "$HTTP_TIMEOUT_SECS" ]] && DELTA_CMD+=(--http-timeout-secs "$HTTP_TIMEOUT_SECS")
+[[ -n "$RSYNC_TIMEOUT_SECS" ]] && DELTA_CMD+=(--rsync-timeout-secs "$RSYNC_TIMEOUT_SECS")
+[[ -n "$RSYNC_MIRROR_ROOT" ]] && DELTA_CMD+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
+[[ -n "$MAX_DEPTH" ]] && DELTA_CMD+=(--max-depth "$MAX_DEPTH")
+[[ -n "$MAX_INSTANCES" ]] && DELTA_CMD+=(--max-instances "$MAX_INSTANCES")
+[[ -n "$TRUST_ANCHOR" ]] && DELTA_CMD+=(--trust-anchor "$TRUST_ANCHOR")
+"${DELTA_CMD[@]}"
+
+mv "$STAGE_DELTA" "$OUT_DIR"
+rm -rf "$STAGE_BASE"
+
+echo "$OUT_DIR"
--- a/scripts/replay_bundle/run_live_bundle_record_multi_rir.sh
+++ b/scripts/replay_bundle/run_live_bundle_record_multi_rir.sh
@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+RIRS=""
+OUT_ROOT=""
+BASE_VALIDATION_TIME=""
+DELTA_VALIDATION_TIME=""
+HTTP_TIMEOUT_SECS=""
+RSYNC_TIMEOUT_SECS=""
+RSYNC_MIRROR_ROOT=""
+MAX_DEPTH=""
+MAX_INSTANCES=""
+NO_BUILD=0
+BIN_DIR="target/release"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/replay_bundle/run_live_bundle_record_multi_rir.sh \
+    --rir <afrinic,apnic,...> \
+    [--out-root <path>] \
+    [--base-validation-time <rfc3339>] \
+    [--delta-validation-time <rfc3339>] \
+    [--http-timeout-secs <n>] \
+    [--rsync-timeout-secs <n>] \
+    [--rsync-mirror-root <path>] \
+    [--max-depth <n>] \
+    [--max-instances <n>] \
+    [--bin-dir <path>] \
+    [--no-build]
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir) RIRS="${2:?}"; shift 2 ;;
+    --out-root) OUT_ROOT="${2:?}"; shift 2 ;;
+    --base-validation-time) BASE_VALIDATION_TIME="${2:?}"; shift 2 ;;
+    --delta-validation-time) DELTA_VALIDATION_TIME="${2:?}"; shift 2 ;;
+    --http-timeout-secs) HTTP_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-timeout-secs) RSYNC_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-mirror-root) RSYNC_MIRROR_ROOT="${2:?}"; shift 2 ;;
+    --max-depth) MAX_DEPTH="${2:?}"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="${2:?}"; shift 2 ;;
+    --bin-dir) BIN_DIR="${2:?}"; shift 2 ;;
+    --no-build) NO_BUILD=1; shift ;;
+    --help|-h) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$RIRS" ]]; then
+  usage >&2
+  exit 2
+fi
+
+RUN_TAG="$(date -u +%Y%m%dT%H%M%SZ)"
+if [[ -z "$OUT_ROOT" ]]; then
+  OUT_ROOT="target/replay/live_bundle_matrix_${RUN_TAG}"
+fi
+mkdir -p "$OUT_ROOT"
+
+resolve_tal_path() {
+  case "$1" in
+    afrinic) printf 'tests/fixtures/tal/afrinic.tal' ;;
+    apnic) printf 'tests/fixtures/tal/apnic-rfc7730-https.tal' ;;
+    arin) printf 'tests/fixtures/tal/arin.tal' ;;
+    lacnic) printf 'tests/fixtures/tal/lacnic.tal' ;;
+    ripe) printf 'tests/fixtures/tal/ripe-ncc.tal' ;;
+    *) echo "unsupported rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+resolve_ta_path() {
+  case "$1" in
+    afrinic) printf 'tests/fixtures/ta/afrinic-ta.cer' ;;
+    apnic) printf 'tests/fixtures/ta/apnic-ta.cer' ;;
+    arin) printf 'tests/fixtures/ta/arin-ta.cer' ;;
+    lacnic) printf 'tests/fixtures/ta/lacnic-ta.cer' ;;
+    ripe) printf 'tests/fixtures/ta/ripe-ncc-ta.cer' ;;
+    *) echo "unsupported rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+SUMMARY_JSON="$OUT_ROOT/summary.json"
+SUMMARY_MD="$OUT_ROOT/summary.md"
+
+python3 - "$SUMMARY_JSON" "$RUN_TAG" <<'PY'
+import json, sys
+out, run_tag = sys.argv[1:]
+with open(out, "w") as fh:
+    json.dump({"runTag": run_tag, "results": []}, fh, indent=2)
+PY
+
+IFS=',' read -r -a RIR_LIST <<< "$RIRS"
+for raw_rir in "${RIR_LIST[@]}"; do
+  rir="$(printf '%s' "$raw_rir" | tr '[:upper:]' '[:lower:]' | xargs)"
+  [[ -n "$rir" ]] || continue
+  tal_path="$(resolve_tal_path "$rir")"
+  ta_path="$(resolve_ta_path "$rir")"
+  out_dir="$OUT_ROOT/${rir}_live_bundle_${RUN_TAG}"
+  cmd=(
+    ./scripts/replay_bundle/run_live_bundle_record.sh
+    --rir "$rir"
+    --out-dir "$out_dir"
+    --tal-path "$tal_path"
+    --ta-path "$ta_path"
+    --trust-anchor "$rir"
+    --bin-dir "$BIN_DIR"
+  )
+  [[ -n "$BASE_VALIDATION_TIME" ]] && cmd+=(--base-validation-time "$BASE_VALIDATION_TIME")
+  [[ -n "$DELTA_VALIDATION_TIME" ]] && cmd+=(--delta-validation-time "$DELTA_VALIDATION_TIME")
+  [[ -n "$HTTP_TIMEOUT_SECS" ]] && cmd+=(--http-timeout-secs "$HTTP_TIMEOUT_SECS")
+  [[ -n "$RSYNC_TIMEOUT_SECS" ]] && cmd+=(--rsync-timeout-secs "$RSYNC_TIMEOUT_SECS")
+  [[ -n "$RSYNC_MIRROR_ROOT" ]] && cmd+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
+  [[ -n "$MAX_DEPTH" ]] && cmd+=(--max-depth "$MAX_DEPTH")
+  [[ -n "$MAX_INSTANCES" ]] && cmd+=(--max-instances "$MAX_INSTANCES")
+  [[ "$NO_BUILD" -eq 1 ]] && cmd+=(--no-build)
+  "${cmd[@]}"
+
+  python3 - "$SUMMARY_JSON" "$rir" "$out_dir" <<'PY'
+import json, pathlib, sys
+summary_path, rir, out_dir = sys.argv[1:]
+summary = json.loads(pathlib.Path(summary_path).read_text())
+bundle = json.loads(pathlib.Path(out_dir, rir, "bundle.json").read_text())
+verification = json.loads(pathlib.Path(out_dir, rir, "verification.json").read_text())
+summary["results"].append({
+    "rir": rir,
+    "outDir": out_dir,
+    "baseVrpCount": bundle["baseVrpCount"],
+    "deltaVrpCount": bundle["deltaVrpCount"],
+    "baseVapCount": bundle["baseVapCount"],
+    "deltaVapCount": bundle["deltaVapCount"],
+    "baseSelfReplayOk": verification["base"]["capture"]["selfReplayOk"],
+    "deltaSelfReplayOk": verification["delta"]["capture"]["selfReplayOk"],
+})
+pathlib.Path(summary_path).write_text(json.dumps(summary, indent=2))
+PY
+done
+
+python3 - "$SUMMARY_JSON" "$SUMMARY_MD" <<'PY'
+import json, pathlib, sys
+summary = json.loads(pathlib.Path(sys.argv[1]).read_text())
+out = pathlib.Path(sys.argv[2])
+lines = [
+    "# Multi-RIR Live Bundle Record Summary",
+    "",
+    f"- runTag: `{summary['runTag']}`",
+    "",
+    "| rir | base_vrps | delta_vrps | base_vaps | delta_vaps | base_self_replay | delta_self_replay | out_dir |",
+    "|---|---:|---:|---:|---:|---|---|---|",
+]
+for item in summary["results"]:
+    lines.append(
+        f"| {item['rir']} | {item['baseVrpCount']} | {item['deltaVrpCount']} | "
+        f"{item['baseVapCount']} | {item['deltaVapCount']} | "
+        f"{str(item['baseSelfReplayOk']).lower()} | {str(item['deltaSelfReplayOk']).lower()} | "
+        f"`{item['outDir']}` |"
+    )
+out.write_text("\n".join(lines) + "\n")
+PY
+
+echo "$OUT_ROOT"
--- a/scripts/replay_verify/README.md
+++ b/scripts/replay_verify/README.md
@@ -43,3 +43,25 @@
 - 同次执行总汇总：
  - `multi_rir_ccr_replay_verify_<timestamp>_summary.md`
  - `multi_rir_ccr_replay_verify_<timestamp>_summary.json`
+
+## `run_peer_bundle_matrix.sh`
+
+用途：
+- 对一组 `ours live bundle` 做本地 peer replay 矩阵验证
+- Routinator 与 `rpki-client` 分别消费相同 bundle root
+- 汇总 `VRP + VAP` 的 base / delta 结果
+
+用法：
+- `./scripts/replay_verify/run_peer_bundle_matrix.sh --bundle-root target/replay/live_bundle_matrix_<timestamp>`
+- `./scripts/replay_verify/run_peer_bundle_matrix.sh --bundle-root target/replay/live_bundle_matrix_<timestamp> --rir apnic,ripe`
+
+主要产物：
+- 输出根目录：
+  - `target/replay/peer_bundle_matrix_<timestamp>/`
+- Routinator：
+  - `target/replay/peer_bundle_matrix_<timestamp>/routinator/<rir>/`
+- `rpki-client`：
+  - `target/replay/peer_bundle_matrix_<timestamp>/rpki-client/`
+- 汇总：
+  - `summary.json`
+  - `summary.md`
--- a/scripts/replay_verify/run_peer_bundle_matrix.sh
+++ b/scripts/replay_verify/run_peer_bundle_matrix.sh
@@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+BUNDLE_ROOT=""
+RIRS=""
+OUT_ROOT=""
+ROUTINATOR_ROOT="/home/yuyr/dev/rust_playground/routinator"
+RPKI_CLIENT_ROOT="/home/yuyr/dev/rpki-client-9.7"
+RPKI_CLIENT_BUILD_DIR="/home/yuyr/dev/rpki-client-9.7/build-m5"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/replay_verify/run_peer_bundle_matrix.sh \
+    --bundle-root <dir> \
+    [--rir <afrinic,apnic,...>] \
+    [--out-root <dir>] \
+    [--routinator-root <dir>] \
+    [--rpki-client-root <dir>] \
+    [--rpki-client-build-dir <dir>]
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --bundle-root) BUNDLE_ROOT="${2:?}"; shift 2 ;;
+    --rir) RIRS="${2:?}"; shift 2 ;;
+    --out-root) OUT_ROOT="${2:?}"; shift 2 ;;
+    --routinator-root) ROUTINATOR_ROOT="${2:?}"; shift 2 ;;
+    --rpki-client-root) RPKI_CLIENT_ROOT="${2:?}"; shift 2 ;;
+    --rpki-client-build-dir) RPKI_CLIENT_BUILD_DIR="${2:?}"; shift 2 ;;
+    --help|-h) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$BUNDLE_ROOT" ]]; then
+  usage >&2
+  exit 2
+fi
+
+RUN_TAG="$(date -u +%Y%m%dT%H%M%SZ)"
+if [[ -z "$OUT_ROOT" ]]; then
+  OUT_ROOT="target/replay/peer_bundle_matrix_${RUN_TAG}"
+fi
+mkdir -p "$OUT_ROOT"
+
+discover_rirs() {
+  python3 - "$BUNDLE_ROOT" <<'PY'
+from pathlib import Path
+import sys
+root = Path(sys.argv[1])
+if (root / "base-locks.json").exists():
+    print(root.name)
+    raise SystemExit
+rirs = []
+for entry in sorted(root.iterdir()):
+    if not entry.is_dir():
+        continue
+    if (entry / "base-locks.json").exists():
+        rirs.append(entry.name)
+        continue
+    nested = sorted(
+        child.name for child in entry.iterdir()
+        if child.is_dir() and (child / "base-locks.json").exists()
+    )
+    if len(nested) == 1:
+        rirs.append(nested[0])
+print(",".join(rirs))
+PY
+}
+
+if [[ -z "$RIRS" ]]; then
+  RIRS="$(discover_rirs)"
+fi
+
+ROUTI_OUT="$OUT_ROOT/routinator"
+CLIENT_OUT="$OUT_ROOT/rpki-client"
+NORMALIZED_BUNDLE_ROOT="$OUT_ROOT/.normalized-bundle-root"
+mkdir -p "$ROUTI_OUT" "$CLIENT_OUT"
+rm -rf "$NORMALIZED_BUNDLE_ROOT"
+mkdir -p "$NORMALIZED_BUNDLE_ROOT"
+
+IFS=',' read -r -a RIR_LIST <<< "$RIRS"
+for raw_rir in "${RIR_LIST[@]}"; do
+  rir="$(printf '%s' "$raw_rir" | tr '[:upper:]' '[:lower:]' | xargs)"
+  [[ -n "$rir" ]] || continue
+  source_bundle_dir=""
+  if [[ -d "$BUNDLE_ROOT/$rir" && -f "$BUNDLE_ROOT/$rir/base-locks.json" ]]; then
+    source_bundle_dir="$BUNDLE_ROOT/$rir"
+  else
+    match="$(find "$BUNDLE_ROOT" -maxdepth 2 -type d -path "*/${rir}" -exec test -f '{}/base-locks.json' ';' -print | head -n 1)"
+    if [[ -z "$match" ]]; then
+      echo "unable to resolve bundle directory for RIR: $rir" >&2
+      exit 1
+    fi
+    source_bundle_dir="$match"
+  fi
+  ln -sfn "$source_bundle_dir" "$NORMALIZED_BUNDLE_ROOT/$rir"
+  "$ROUTINATOR_ROOT/bench/multi_rir_demo_ours/run_single_rir_ours_bundle.sh" \
+    "$source_bundle_dir" \
+    "$ROUTI_OUT/$rir"
+done
+
+CLIENT_ARGS=(
+  python3 "$RPKI_CLIENT_ROOT/tools/run_bundle_matrix.py"
+  --bundle-dir "$NORMALIZED_BUNDLE_ROOT"
+  --build-dir "$RPKI_CLIENT_BUILD_DIR"
+  --work-dir "$CLIENT_OUT"
+)
+for raw_rir in "${RIR_LIST[@]}"; do
+  rir="$(printf '%s' "$raw_rir" | tr '[:upper:]' '[:lower:]' | xargs)"
+  [[ -n "$rir" ]] || continue
+  CLIENT_ARGS+=(--rir "$rir")
+done
+"${CLIENT_ARGS[@]}"
+
+SUMMARY_JSON="$OUT_ROOT/summary.json"
+SUMMARY_MD="$OUT_ROOT/summary.md"
+
+python3 - "$ROUTI_OUT" "$CLIENT_OUT/matrix-summary.json" "$SUMMARY_JSON" <<'PY'
+import json
+from pathlib import Path
+import sys
+
+routi_root = Path(sys.argv[1])
+client_summary = json.loads(Path(sys.argv[2]).read_text())
+summary_path = Path(sys.argv[3])
+summary = {"routinator": {}, "rpki_client": client_summary}
+for verification in sorted(routi_root.glob("*/verification.json")):
+    rir = verification.parent.name
+    summary["routinator"][rir] = json.loads(verification.read_text())
+summary_path.write_text(json.dumps(summary, indent=2))
+PY
+
+python3 - "$SUMMARY_JSON" "$SUMMARY_MD" <<'PY'
+import json
+from pathlib import Path
+import sys
+summary = json.loads(Path(sys.argv[1]).read_text())
+out = Path(sys.argv[2])
+lines = [
+    "# Peer Bundle Matrix Summary",
+    "",
+    "## Routinator",
+    "",
+    "| rir | base_vrp | delta_vrp | base_vap | delta_vap |",
+    "|---|---|---|---|---|",
+]
+for rir, data in sorted(summary["routinator"].items()):
+    lines.append(
+        f"| {rir} | {str(data.get('baseMatch')).lower()} | {str(data.get('deltaMatch')).lower()} | "
+        f"{str(data.get('baseVapsMatch')).lower()} | {str(data.get('deltaVapsMatch')).lower()} |"
+    )
+lines += [
+    "",
+    "## rpki-client",
+    "",
+    "| rir | base_vrp | delta_vrp | base_vap | delta_vap |",
+    "|---|---|---|---|---|",
+]
+for rir, phases in sorted(summary["rpki_client"].items()):
+    base = phases.get("base", {})
+    delta = phases.get("delta", {})
+    lines.append(
+        f"| {rir} | {str(base.get('match')).lower()} | {str(delta.get('match')).lower()} | "
+        f"{str(base.get('vaps_match')).lower()} | {str(delta.get('vaps_match')).lower()} |"
+    )
+out.write_text("\n".join(lines) + "\n")
+PY
+
+echo "$OUT_ROOT"