20260330 完成live bundle录制，远程录制，以及与routinator/rpki-client replay对比

2026-03-31 17:34:32 +08:00
parent cd0ba15286
commit 6edc420ce2
19 changed files with 4059 additions and 22 deletions
--- a/scripts/replay_bundle/README.md
+++ b/scripts/replay_bundle/README.md
@@ -0,0 +1,94 @@
+# Live Bundle Record
+
+`run_live_bundle_record.sh` 是当前 `ours` 的单命令 live bundle 录制入口。
+
+它做三件事：
+
+1. 联网执行 **live base recorder**
+2. 基于刚录制的 base bundle 执行 **live delta recorder**
+3. 产出一个统一的最终目录，包含：
+   - `base-payload-archive/`
+   - `payload-delta-archive/`
+   - `base-locks.json`
+   - `locks-delta.json`
+   - `tal.tal`
+   - `ta.cer`
+   - `base.ccr`
+   - `delta.ccr`
+   - `base-vrps.csv`
+   - `base-vaps.csv`
+   - `record-delta.csv`
+   - `record-delta-vaps.csv`
+   - `bundle.json`
+   - `verification.json`
+   - `timings/`
+
+## 用法
+
+```bash
+cd rpki
+./scripts/replay_bundle/run_live_bundle_record.sh \
+  --rir apnic \
+  --tal-path tests/fixtures/tal/apnic-rfc7730-https.tal \
+  --ta-path tests/fixtures/ta/apnic-ta.cer
+```
+
+默认输出目录：
+
+```text
+target/replay/<rir>_live_bundle_<timestamp>
+```
+
+如果要一次录制多个 RIR，使用：
+
+```bash
+cd rpki
+./scripts/replay_bundle/run_live_bundle_record_multi_rir.sh \
+  --rir afrinic,apnic,arin,lacnic,ripe
+```
+
+默认输出目录：
+
+```text
+target/replay/live_bundle_matrix_<timestamp>
+```
+
+每个 RIR 会落到：
+
+```text
+target/replay/live_bundle_matrix_<timestamp>/<rir>_live_bundle_<timestamp>
+```
+
+## 可选参数
+
+- `--out-dir <path>`
+- `--base-validation-time <rfc3339>`
+- `--delta-validation-time <rfc3339>`
+- `--http-timeout-secs <n>`
+- `--rsync-timeout-secs <n>`
+- `--rsync-mirror-root <path>`
+- `--max-depth <n>`
+- `--max-instances <n>`
+- `--trust-anchor <name>`
+- `--bin-dir <path>`
+- `--no-build`
+
+`run_live_bundle_record_multi_rir.sh` 会自动按 RIR 选择当前仓库内置的：
+
+- `tests/fixtures/tal/*.tal`
+- `tests/fixtures/ta/*.cer`
+
+并将 `--trust-anchor` 设置为对应 RIR 名称。
+
+## 说明
+
+- 该脚本会先构建：
+  - `replay_bundle_capture`
+  - `replay_bundle_capture_delta`
+- 如果提供 `--no-build`，则直接复用：
+  - `--bin-dir <path>` 下的现有二进制
+- 中间 staging 目录：
+  - `<out>.stage-base`
+  - `<out>.stage-delta`
+  在成功完成后会清理，只保留最终输出目录。
+- 最终输出目录是 **delta 阶段产物**，其中已经包含 base 阶段结果。
--- a/scripts/replay_bundle/run_live_bundle_record.sh
+++ b/scripts/replay_bundle/run_live_bundle_record.sh
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+RIR=""
+OUT_DIR=""
+TAL_PATH=""
+TA_PATH=""
+BASE_VALIDATION_TIME=""
+DELTA_VALIDATION_TIME=""
+HTTP_TIMEOUT_SECS=""
+RSYNC_TIMEOUT_SECS=""
+RSYNC_MIRROR_ROOT=""
+MAX_DEPTH=""
+MAX_INSTANCES=""
+TRUST_ANCHOR=""
+NO_BUILD=0
+BIN_DIR="target/release"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/replay_bundle/run_live_bundle_record.sh \
+    --rir <name> \
+    --tal-path <path> \
+    --ta-path <path> \
+    [--out-dir <path>] \
+    [--base-validation-time <rfc3339>] \
+    [--delta-validation-time <rfc3339>] \
+    [--http-timeout-secs <n>] \
+    [--rsync-timeout-secs <n>] \
+    [--rsync-mirror-root <path>] \
+    [--max-depth <n>] \
+    [--max-instances <n>] \
+    [--trust-anchor <name>] \
+    [--bin-dir <path>] \
+    [--no-build]
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir) RIR="${2:?}"; shift 2 ;;
+    --out-dir) OUT_DIR="${2:?}"; shift 2 ;;
+    --tal-path) TAL_PATH="${2:?}"; shift 2 ;;
+    --ta-path) TA_PATH="${2:?}"; shift 2 ;;
+    --base-validation-time) BASE_VALIDATION_TIME="${2:?}"; shift 2 ;;
+    --delta-validation-time) DELTA_VALIDATION_TIME="${2:?}"; shift 2 ;;
+    --http-timeout-secs) HTTP_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-timeout-secs) RSYNC_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-mirror-root) RSYNC_MIRROR_ROOT="${2:?}"; shift 2 ;;
+    --max-depth) MAX_DEPTH="${2:?}"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="${2:?}"; shift 2 ;;
+    --trust-anchor) TRUST_ANCHOR="${2:?}"; shift 2 ;;
+    --bin-dir) BIN_DIR="${2:?}"; shift 2 ;;
+    --no-build) NO_BUILD=1; shift ;;
+    --help|-h) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$RIR" || -z "$TAL_PATH" || -z "$TA_PATH" ]]; then
+  usage >&2
+  exit 2
+fi
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+if [[ -z "$OUT_DIR" ]]; then
+  OUT_DIR="target/replay/${RIR}_live_bundle_${TS}"
+fi
+
+STAGE_BASE="${OUT_DIR}.stage-base"
+STAGE_DELTA="${OUT_DIR}.stage-delta"
+
+rm -rf "$OUT_DIR" "$STAGE_BASE" "$STAGE_DELTA"
+mkdir -p "$(dirname "$OUT_DIR")"
+
+CAPTURE_BIN="$BIN_DIR/replay_bundle_capture"
+DELTA_CAPTURE_BIN="$BIN_DIR/replay_bundle_capture_delta"
+
+if [[ "$NO_BUILD" -eq 0 ]]; then
+  echo "[1/3] build release binaries"
+  cargo build --release --bin replay_bundle_capture --bin replay_bundle_capture_delta
+else
+  echo "[1/3] reuse existing binaries from $BIN_DIR"
+fi
+
+if [[ ! -x "$CAPTURE_BIN" ]]; then
+  echo "missing executable: $CAPTURE_BIN" >&2
+  exit 1
+fi
+if [[ ! -x "$DELTA_CAPTURE_BIN" ]]; then
+  echo "missing executable: $DELTA_CAPTURE_BIN" >&2
+  exit 1
+fi
+
+echo "[2/3] record live base bundle into $STAGE_BASE"
+BASE_CMD=(
+  "$CAPTURE_BIN"
+  --rir "$RIR"
+  --out-dir "$STAGE_BASE"
+  --tal-path "$TAL_PATH"
+  --ta-path "$TA_PATH"
+)
+[[ -n "$BASE_VALIDATION_TIME" ]] && BASE_CMD+=(--validation-time "$BASE_VALIDATION_TIME")
+[[ -n "$HTTP_TIMEOUT_SECS" ]] && BASE_CMD+=(--http-timeout-secs "$HTTP_TIMEOUT_SECS")
+[[ -n "$RSYNC_TIMEOUT_SECS" ]] && BASE_CMD+=(--rsync-timeout-secs "$RSYNC_TIMEOUT_SECS")
+[[ -n "$RSYNC_MIRROR_ROOT" ]] && BASE_CMD+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
+[[ -n "$MAX_DEPTH" ]] && BASE_CMD+=(--max-depth "$MAX_DEPTH")
+[[ -n "$MAX_INSTANCES" ]] && BASE_CMD+=(--max-instances "$MAX_INSTANCES")
+[[ -n "$TRUST_ANCHOR" ]] && BASE_CMD+=(--trust-anchor "$TRUST_ANCHOR")
+"${BASE_CMD[@]}"
+
+echo "[3/3] record live delta bundle into $STAGE_DELTA"
+DELTA_CMD=(
+  "$DELTA_CAPTURE_BIN"
+  --rir "$RIR"
+  --base-bundle-dir "$STAGE_BASE"
+  --out-dir "$STAGE_DELTA"
+)
+[[ -n "$DELTA_VALIDATION_TIME" ]] && DELTA_CMD+=(--validation-time "$DELTA_VALIDATION_TIME")
+[[ -n "$HTTP_TIMEOUT_SECS" ]] && DELTA_CMD+=(--http-timeout-secs "$HTTP_TIMEOUT_SECS")
+[[ -n "$RSYNC_TIMEOUT_SECS" ]] && DELTA_CMD+=(--rsync-timeout-secs "$RSYNC_TIMEOUT_SECS")
+[[ -n "$RSYNC_MIRROR_ROOT" ]] && DELTA_CMD+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
+[[ -n "$MAX_DEPTH" ]] && DELTA_CMD+=(--max-depth "$MAX_DEPTH")
+[[ -n "$MAX_INSTANCES" ]] && DELTA_CMD+=(--max-instances "$MAX_INSTANCES")
+[[ -n "$TRUST_ANCHOR" ]] && DELTA_CMD+=(--trust-anchor "$TRUST_ANCHOR")
+"${DELTA_CMD[@]}"
+
+mv "$STAGE_DELTA" "$OUT_DIR"
+rm -rf "$STAGE_BASE"
+
+echo "$OUT_DIR"
--- a/scripts/replay_bundle/run_live_bundle_record_multi_rir.sh
+++ b/scripts/replay_bundle/run_live_bundle_record_multi_rir.sh
@@ -0,0 +1,166 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+RIRS=""
+OUT_ROOT=""
+BASE_VALIDATION_TIME=""
+DELTA_VALIDATION_TIME=""
+HTTP_TIMEOUT_SECS=""
+RSYNC_TIMEOUT_SECS=""
+RSYNC_MIRROR_ROOT=""
+MAX_DEPTH=""
+MAX_INSTANCES=""
+NO_BUILD=0
+BIN_DIR="target/release"
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/replay_bundle/run_live_bundle_record_multi_rir.sh \
+    --rir <afrinic,apnic,...> \
+    [--out-root <path>] \
+    [--base-validation-time <rfc3339>] \
+    [--delta-validation-time <rfc3339>] \
+    [--http-timeout-secs <n>] \
+    [--rsync-timeout-secs <n>] \
+    [--rsync-mirror-root <path>] \
+    [--max-depth <n>] \
+    [--max-instances <n>] \
+    [--bin-dir <path>] \
+    [--no-build]
+EOF
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rir) RIRS="${2:?}"; shift 2 ;;
+    --out-root) OUT_ROOT="${2:?}"; shift 2 ;;
+    --base-validation-time) BASE_VALIDATION_TIME="${2:?}"; shift 2 ;;
+    --delta-validation-time) DELTA_VALIDATION_TIME="${2:?}"; shift 2 ;;
+    --http-timeout-secs) HTTP_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-timeout-secs) RSYNC_TIMEOUT_SECS="${2:?}"; shift 2 ;;
+    --rsync-mirror-root) RSYNC_MIRROR_ROOT="${2:?}"; shift 2 ;;
+    --max-depth) MAX_DEPTH="${2:?}"; shift 2 ;;
+    --max-instances) MAX_INSTANCES="${2:?}"; shift 2 ;;
+    --bin-dir) BIN_DIR="${2:?}"; shift 2 ;;
+    --no-build) NO_BUILD=1; shift ;;
+    --help|-h) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$RIRS" ]]; then
+  usage >&2
+  exit 2
+fi
+
+RUN_TAG="$(date -u +%Y%m%dT%H%M%SZ)"
+if [[ -z "$OUT_ROOT" ]]; then
+  OUT_ROOT="target/replay/live_bundle_matrix_${RUN_TAG}"
+fi
+mkdir -p "$OUT_ROOT"
+
+resolve_tal_path() {
+  case "$1" in
+    afrinic) printf 'tests/fixtures/tal/afrinic.tal' ;;
+    apnic) printf 'tests/fixtures/tal/apnic-rfc7730-https.tal' ;;
+    arin) printf 'tests/fixtures/tal/arin.tal' ;;
+    lacnic) printf 'tests/fixtures/tal/lacnic.tal' ;;
+    ripe) printf 'tests/fixtures/tal/ripe-ncc.tal' ;;
+    *) echo "unsupported rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+resolve_ta_path() {
+  case "$1" in
+    afrinic) printf 'tests/fixtures/ta/afrinic-ta.cer' ;;
+    apnic) printf 'tests/fixtures/ta/apnic-ta.cer' ;;
+    arin) printf 'tests/fixtures/ta/arin-ta.cer' ;;
+    lacnic) printf 'tests/fixtures/ta/lacnic-ta.cer' ;;
+    ripe) printf 'tests/fixtures/ta/ripe-ncc-ta.cer' ;;
+    *) echo "unsupported rir: $1" >&2; exit 2 ;;
+  esac
+}
+
+SUMMARY_JSON="$OUT_ROOT/summary.json"
+SUMMARY_MD="$OUT_ROOT/summary.md"
+
+python3 - "$SUMMARY_JSON" "$RUN_TAG" <<'PY'
+import json, sys
+out, run_tag = sys.argv[1:]
+with open(out, "w") as fh:
+    json.dump({"runTag": run_tag, "results": []}, fh, indent=2)
+PY
+
+IFS=',' read -r -a RIR_LIST <<< "$RIRS"
+for raw_rir in "${RIR_LIST[@]}"; do
+  rir="$(printf '%s' "$raw_rir" | tr '[:upper:]' '[:lower:]' | xargs)"
+  [[ -n "$rir" ]] || continue
+  tal_path="$(resolve_tal_path "$rir")"
+  ta_path="$(resolve_ta_path "$rir")"
+  out_dir="$OUT_ROOT/${rir}_live_bundle_${RUN_TAG}"
+  cmd=(
+    ./scripts/replay_bundle/run_live_bundle_record.sh
+    --rir "$rir"
+    --out-dir "$out_dir"
+    --tal-path "$tal_path"
+    --ta-path "$ta_path"
+    --trust-anchor "$rir"
+    --bin-dir "$BIN_DIR"
+  )
+  [[ -n "$BASE_VALIDATION_TIME" ]] && cmd+=(--base-validation-time "$BASE_VALIDATION_TIME")
+  [[ -n "$DELTA_VALIDATION_TIME" ]] && cmd+=(--delta-validation-time "$DELTA_VALIDATION_TIME")
+  [[ -n "$HTTP_TIMEOUT_SECS" ]] && cmd+=(--http-timeout-secs "$HTTP_TIMEOUT_SECS")
+  [[ -n "$RSYNC_TIMEOUT_SECS" ]] && cmd+=(--rsync-timeout-secs "$RSYNC_TIMEOUT_SECS")
+  [[ -n "$RSYNC_MIRROR_ROOT" ]] && cmd+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
+  [[ -n "$MAX_DEPTH" ]] && cmd+=(--max-depth "$MAX_DEPTH")
+  [[ -n "$MAX_INSTANCES" ]] && cmd+=(--max-instances "$MAX_INSTANCES")
+  [[ "$NO_BUILD" -eq 1 ]] && cmd+=(--no-build)
+  "${cmd[@]}"
+
+  python3 - "$SUMMARY_JSON" "$rir" "$out_dir" <<'PY'
+import json, pathlib, sys
+summary_path, rir, out_dir = sys.argv[1:]
+summary = json.loads(pathlib.Path(summary_path).read_text())
+bundle = json.loads(pathlib.Path(out_dir, rir, "bundle.json").read_text())
+verification = json.loads(pathlib.Path(out_dir, rir, "verification.json").read_text())
+summary["results"].append({
+    "rir": rir,
+    "outDir": out_dir,
+    "baseVrpCount": bundle["baseVrpCount"],
+    "deltaVrpCount": bundle["deltaVrpCount"],
+    "baseVapCount": bundle["baseVapCount"],
+    "deltaVapCount": bundle["deltaVapCount"],
+    "baseSelfReplayOk": verification["base"]["capture"]["selfReplayOk"],
+    "deltaSelfReplayOk": verification["delta"]["capture"]["selfReplayOk"],
+})
+pathlib.Path(summary_path).write_text(json.dumps(summary, indent=2))
+PY
+done
+
+python3 - "$SUMMARY_JSON" "$SUMMARY_MD" <<'PY'
+import json, pathlib, sys
+summary = json.loads(pathlib.Path(sys.argv[1]).read_text())
+out = pathlib.Path(sys.argv[2])
+lines = [
+    "# Multi-RIR Live Bundle Record Summary",
+    "",
+    f"- runTag: `{summary['runTag']}`",
+    "",
+    "| rir | base_vrps | delta_vrps | base_vaps | delta_vaps | base_self_replay | delta_self_replay | out_dir |",
+    "|---|---:|---:|---:|---:|---|---|---|",
+]
+for item in summary["results"]:
+    lines.append(
+        f"| {item['rir']} | {item['baseVrpCount']} | {item['deltaVrpCount']} | "
+        f"{item['baseVapCount']} | {item['deltaVapCount']} | "
+        f"{str(item['baseSelfReplayOk']).lower()} | {str(item['deltaSelfReplayOk']).lower()} | "
+        f"`{item['outDir']}` |"
+    )
+out.write_text("\n".join(lines) + "\n")
+PY
+
+echo "$OUT_ROOT"