rpki/scripts/cir/run_cir_replay_matrix.sh

#!/usr/bin/env bash
set -euo pipefail

usage() {
  cat <<'EOF'
Usage:
  ./scripts/cir/run_cir_replay_matrix.sh \
    --cir <path> \
    --static-root <path> \
    --out-dir <path> \
    --reference-ccr <path> \
    --rpki-client-build-dir <path> \
    [--keep-db] \
    [--rpki-bin <path>] \
    [--routinator-root <path>] \
    [--routinator-bin <path>] \
    [--real-rsync-bin <path>]
EOF
}

ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"

CIR=""
STATIC_ROOT=""
OUT_DIR=""
REFERENCE_CCR=""
RPKI_CLIENT_BUILD_DIR=""
KEEP_DB=0
RPKI_BIN="${RPKI_BIN:-$ROOT_DIR/target/release/rpki}"
ROUTINATOR_ROOT="${ROUTINATOR_ROOT:-/home/yuyr/dev/rust_playground/routinator}"
ROUTINATOR_BIN="${ROUTINATOR_BIN:-$ROUTINATOR_ROOT/target/debug/routinator}"
REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
OURS_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_ours.sh"
ROUTINATOR_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_routinator.sh"
RPKI_CLIENT_SCRIPT="$ROOT_DIR/scripts/cir/run_cir_replay_rpki_client.sh"

while [[ $# -gt 0 ]]; do
  case "$1" in
    --cir) CIR="$2"; shift 2 ;;
    --static-root) STATIC_ROOT="$2"; shift 2 ;;
    --out-dir) OUT_DIR="$2"; shift 2 ;;
    --reference-ccr) REFERENCE_CCR="$2"; shift 2 ;;
    --rpki-client-build-dir) RPKI_CLIENT_BUILD_DIR="$2"; shift 2 ;;
    --keep-db) KEEP_DB=1; shift ;;
    --rpki-bin) RPKI_BIN="$2"; shift 2 ;;
    --routinator-root) ROUTINATOR_ROOT="$2"; shift 2 ;;
    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
    -h|--help) usage; exit 0 ;;
    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
  esac
done

[[ -n "$CIR" && -n "$STATIC_ROOT" && -n "$OUT_DIR" && -n "$REFERENCE_CCR" && -n "$RPKI_CLIENT_BUILD_DIR" ]] || {
  usage >&2
  exit 2
}

mkdir -p "$OUT_DIR"

run_with_timing() {
  local summary_path="$1"
  local timing_path="$2"
  shift 2
  local start end status
  start="$(python3 - <<'PY'
import time
print(time.perf_counter_ns())
PY
)"
  if "$@"; then
    status=0
  else
    status=$?
  fi
  end="$(python3 - <<'PY'
import time
print(time.perf_counter_ns())
PY
)"
  python3 - <<'PY' "$summary_path" "$timing_path" "$status" "$start" "$end"
import json, sys
summary_path, timing_path, status, start, end = sys.argv[1:]
duration_ms = max(0, (int(end) - int(start)) // 1_000_000)
data = {"exitCode": int(status), "durationMs": duration_ms}
try:
    with open(summary_path, "r", encoding="utf-8") as f:
        data["compare"] = json.load(f)
except FileNotFoundError:
    data["compare"] = None
with open(timing_path, "w", encoding="utf-8") as f:
    json.dump(data, f, indent=2)
PY
  return "$status"
}

OURS_OUT="$OUT_DIR/ours"
ROUTINATOR_OUT="$OUT_DIR/routinator"
RPKI_CLIENT_OUT="$OUT_DIR/rpki-client"
mkdir -p "$OURS_OUT" "$ROUTINATOR_OUT" "$RPKI_CLIENT_OUT"

ours_cmd=(
  "$OURS_SCRIPT"
  --cir "$CIR"
  --static-root "$STATIC_ROOT"
  --out-dir "$OURS_OUT"
  --reference-ccr "$REFERENCE_CCR"
  --rpki-bin "$RPKI_BIN"
  --real-rsync-bin "$REAL_RSYNC_BIN"
)
routinator_cmd=(
  "$ROUTINATOR_SCRIPT"
  --cir "$CIR"
  --static-root "$STATIC_ROOT"
  --out-dir "$ROUTINATOR_OUT"
  --reference-ccr "$REFERENCE_CCR"
  --routinator-root "$ROUTINATOR_ROOT"
  --routinator-bin "$ROUTINATOR_BIN"
  --real-rsync-bin "$REAL_RSYNC_BIN"
)
rpki_client_cmd=(
  "$RPKI_CLIENT_SCRIPT"
  --cir "$CIR"
  --static-root "$STATIC_ROOT"
  --out-dir "$RPKI_CLIENT_OUT"
  --reference-ccr "$REFERENCE_CCR"
  --build-dir "$RPKI_CLIENT_BUILD_DIR"
  --real-rsync-bin "$REAL_RSYNC_BIN"
)

if [[ "$KEEP_DB" -eq 1 ]]; then
  ours_cmd+=(--keep-db)
  routinator_cmd+=(--keep-db)
  rpki_client_cmd+=(--keep-db)
fi

ours_status=0
routinator_status=0
rpki_client_status=0
if run_with_timing "$OURS_OUT/compare-summary.json" "$OURS_OUT/timing.json" "${ours_cmd[@]}"; then
  :
else
  ours_status=$?
fi
if run_with_timing "$ROUTINATOR_OUT/compare-summary.json" "$ROUTINATOR_OUT/timing.json" "${routinator_cmd[@]}"; then
  :
else
  routinator_status=$?
fi
if run_with_timing "$RPKI_CLIENT_OUT/compare-summary.json" "$RPKI_CLIENT_OUT/timing.json" "${rpki_client_cmd[@]}"; then
  :
else
  rpki_client_status=$?
fi

SUMMARY_JSON="$OUT_DIR/summary.json"
SUMMARY_MD="$OUT_DIR/summary.md"
DETAIL_MD="$OUT_DIR/detail.md"

python3 - <<'PY' \
  "$CIR" \
  "$STATIC_ROOT" \
  "$REFERENCE_CCR" \
  "$OURS_OUT" \
  "$ROUTINATOR_OUT" \
  "$RPKI_CLIENT_OUT" \
  "$SUMMARY_JSON" \
  "$SUMMARY_MD" \
  "$DETAIL_MD"
import json
import sys
from pathlib import Path

cir_path, static_root, reference_ccr, ours_out, routinator_out, rpki_client_out, summary_json, summary_md, detail_md = sys.argv[1:]

participants = []
all_match = True
for name, out_dir in [
    ("ours", ours_out),
    ("routinator", routinator_out),
    ("rpki-client", rpki_client_out),
]:
    out = Path(out_dir)
    timing = json.loads((out / "timing.json").read_text(encoding="utf-8"))
    compare = timing.get("compare") or {}
    vrps = compare.get("vrps") or {}
    vaps = compare.get("vaps") or {}
    participant = {
        "name": name,
        "outDir": str(out),
        "tmpRoot": str(out / ".tmp"),
        "mirrorPath": str(out / ".tmp" / "mirror"),
        "timingPath": str(out / "timing.json"),
        "summaryPath": str(out / "compare-summary.json"),
        "exitCode": timing["exitCode"],
        "durationMs": timing["durationMs"],
        "vrps": vrps,
        "vaps": vaps,
        "match": bool(vrps.get("match")) and bool(vaps.get("match")) and timing["exitCode"] == 0,
        "logPaths": [str(path) for path in sorted(out.glob("*.log"))],
    }
    participants.append(participant)
    all_match = all_match and participant["match"]

summary = {
    "cirPath": cir_path,
    "staticRoot": static_root,
    "referenceCcr": reference_ccr,
    "participants": participants,
    "allMatch": all_match,
}
Path(summary_json).write_text(json.dumps(summary, indent=2), encoding="utf-8")

lines = [
    "# CIR Replay Matrix Summary",
    "",
    f"- `cir`: `{cir_path}`",
    f"- `static_root`: `{static_root}`",
    f"- `reference_ccr`: `{reference_ccr}`",
    f"- `all_match`: `{all_match}`",
    "",
    "| Participant | Exit | Duration (ms) | VRP actual/ref | VRP match | VAP actual/ref | VAP match | Log |",
    "| --- | ---: | ---: | --- | --- | --- | --- | --- |",
]
for participant in participants:
    vrps = participant["vrps"] or {}
    vaps = participant["vaps"] or {}
    log_path = participant["logPaths"][0] if participant["logPaths"] else ""
    lines.append(
        "| {name} | {exit_code} | {duration_ms} | {vrp_actual}/{vrp_ref} | {vrp_match} | {vap_actual}/{vap_ref} | {vap_match} | `{log_path}` |".format(
            name=participant["name"],
            exit_code=participant["exitCode"],
            duration_ms=participant["durationMs"],
            vrp_actual=vrps.get("actual", "-"),
            vrp_ref=vrps.get("reference", "-"),
            vrp_match=vrps.get("match", False),
            vap_actual=vaps.get("actual", "-"),
            vap_ref=vaps.get("reference", "-"),
            vap_match=vaps.get("match", False),
            log_path=log_path,
        )
    )
Path(summary_md).write_text("\n".join(lines) + "\n", encoding="utf-8")

detail_lines = [
    "# CIR Replay Matrix Detail",
    "",
]
for participant in participants:
    vrps = participant["vrps"] or {}
    vaps = participant["vaps"] or {}
    detail_lines.extend([
        f"## {participant['name']}",
        f"- `exit_code`: `{participant['exitCode']}`",
        f"- `duration_ms`: `{participant['durationMs']}`",
        f"- `out_dir`: `{participant['outDir']}`",
        f"- `tmp_root`: `{participant['tmpRoot']}`",
        f"- `mirror_path`: `{participant['mirrorPath']}`",
        f"- `summary_path`: `{participant['summaryPath']}`",
        f"- `timing_path`: `{participant['timingPath']}`",
        f"- `log_paths`: `{', '.join(participant['logPaths'])}`",
        f"- `vrps`: `actual={vrps.get('actual', '-')}` `reference={vrps.get('reference', '-')}` `match={vrps.get('match', False)}`",
        f"- `vaps`: `actual={vaps.get('actual', '-')}` `reference={vaps.get('reference', '-')}` `match={vaps.get('match', False)}`",
        f"- `vrps.only_in_actual`: `{vrps.get('only_in_actual', [])}`",
        f"- `vrps.only_in_reference`: `{vrps.get('only_in_reference', [])}`",
        f"- `vaps.only_in_actual`: `{vaps.get('only_in_actual', [])}`",
        f"- `vaps.only_in_reference`: `{vaps.get('only_in_reference', [])}`",
        "",
    ])
Path(detail_md).write_text("\n".join(detail_lines), encoding="utf-8")
PY

if [[ "$ours_status" -ne 0 || "$routinator_status" -ne 0 || "$rpki_client_status" -ne 0 ]]; then
  exit 1
fi

all_match="$(python3 - <<'PY' "$SUMMARY_JSON"
import json,sys
print("true" if json.load(open(sys.argv[1]))["allMatch"] else "false")
PY
)"
if [[ "$all_match" != "true" ]]; then
  exit 1
fi

echo "done: $OUT_DIR"