优化数据对象decode + profile validate;benchmark对比routinator geomean 0.8
This commit is contained in:
123
scripts/stage2_perf_compare_m4.sh
Executable file
123
scripts/stage2_perf_compare_m4.sh
Executable file
@@ -0,0 +1,123 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# M4: Compare decode+profile (OURS) vs routinator baseline (rpki crate 0.19.1)
|
||||
# on selected_der_v2 fixtures (cer/crl/manifest/roa/aspa).
|
||||
#
|
||||
# Outputs under:
|
||||
# - rpki/target/bench/stage2_selected_der_v2_routinator_decode_release.{csv,md}
|
||||
# - rpki/target/bench/stage2_selected_der_v2_compare_ours_vs_routinator_decode_release.{csv,md}
|
||||
#
|
||||
# Notes:
|
||||
# - OURS decode benchmark is produced by:
|
||||
# `cargo test --release --test bench_stage2_decode_profile_selected_der_v2 -- --ignored --nocapture`
|
||||
# and writes `stage2_selected_der_v2_decode_release.csv` when BENCH_OUT_CSV is set.
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
RPKI_DIR="$ROOT_DIR"
|
||||
|
||||
OUT_DIR="$RPKI_DIR/target/bench"
|
||||
mkdir -p "$OUT_DIR"
|
||||
|
||||
OURS_CSV="${OURS_CSV:-$OUT_DIR/stage2_selected_der_v2_decode_release.csv}"
|
||||
|
||||
ROUT_CSV="${ROUT_CSV:-$OUT_DIR/stage2_selected_der_v2_routinator_decode_release.csv}"
|
||||
ROUT_MD="${ROUT_MD:-$OUT_DIR/stage2_selected_der_v2_routinator_decode_release.md}"
|
||||
|
||||
COMPARE_CSV="${COMPARE_CSV:-$OUT_DIR/stage2_selected_der_v2_compare_ours_vs_routinator_decode_release.csv}"
|
||||
COMPARE_MD="${COMPARE_MD:-$OUT_DIR/stage2_selected_der_v2_compare_ours_vs_routinator_decode_release.md}"
|
||||
|
||||
WARMUP_ITERS="${WARMUP_ITERS:-10}"
|
||||
ROUNDS="${ROUNDS:-3}"
|
||||
MIN_ROUND_MS="${MIN_ROUND_MS:-200}"
|
||||
|
||||
if [[ ! -f "$OURS_CSV" ]]; then
|
||||
echo "ERROR: missing OURS CSV: $OURS_CSV" >&2
|
||||
echo "Hint: run:" >&2
|
||||
echo " cd rpki && BENCH_WARMUP_ITERS=$WARMUP_ITERS BENCH_ROUNDS=$ROUNDS BENCH_MIN_ROUND_MS=$MIN_ROUND_MS \\" >&2
|
||||
echo " BENCH_OUT_CSV=target/bench/stage2_selected_der_v2_decode_release.csv \\" >&2
|
||||
echo " cargo test --release --test bench_stage2_decode_profile_selected_der_v2 -- --ignored --nocapture" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
echo "[1/2] Run routinator baseline bench (release)..." >&2
|
||||
(cd "$RPKI_DIR/benchmark/routinator_object_bench" && cargo run --release -q -- \
|
||||
--dir "$RPKI_DIR/tests/benchmark/selected_der_v2" \
|
||||
--warmup-iters "$WARMUP_ITERS" \
|
||||
--rounds "$ROUNDS" \
|
||||
--min-round-ms "$MIN_ROUND_MS" \
|
||||
--out-csv "$ROUT_CSV" \
|
||||
--out-md "$ROUT_MD")
|
||||
|
||||
echo "[2/2] Join CSVs + compute ratios..." >&2
|
||||
python3 - "$OURS_CSV" "$ROUT_CSV" "$COMPARE_CSV" "$COMPARE_MD" <<'PY'
|
||||
import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ours_path = Path(sys.argv[1])
|
||||
rout_path = Path(sys.argv[2])
|
||||
out_csv_path = Path(sys.argv[3])
|
||||
out_md_path = Path(sys.argv[4])
|
||||
|
||||
def read_csv(path: Path):
|
||||
with path.open(newline="") as f:
|
||||
return list(csv.DictReader(f))
|
||||
|
||||
ours_rows = read_csv(ours_path)
|
||||
rout_rows = read_csv(rout_path)
|
||||
|
||||
rout_by_key = {(r["type"], r["sample"]): r for r in rout_rows}
|
||||
|
||||
out_rows = []
|
||||
for r in ours_rows:
|
||||
key = (r["type"], r["sample"])
|
||||
rr = rout_by_key.get(key)
|
||||
if rr is None:
|
||||
raise SystemExit(f"missing routinator row for {key}")
|
||||
|
||||
ours_ns = float(r["avg_ns_per_op"])
|
||||
rout_ns = float(rr["avg_ns_per_op"])
|
||||
ratio = (ours_ns / rout_ns) if rout_ns != 0.0 else float("inf")
|
||||
|
||||
out_rows.append({
|
||||
"type": r["type"],
|
||||
"sample": r["sample"],
|
||||
"size_bytes": r["size_bytes"],
|
||||
"complexity": r["complexity"],
|
||||
"ours_avg_ns_per_op": f"{ours_ns:.2f}",
|
||||
"ours_ops_per_sec": f"{float(r['ops_per_sec']):.2f}",
|
||||
"rout_avg_ns_per_op": f"{rout_ns:.2f}",
|
||||
"rout_ops_per_sec": f"{float(rr['ops_per_sec']):.2f}",
|
||||
"ratio_ours_over_rout": f"{ratio:.4f}",
|
||||
})
|
||||
|
||||
out_rows.sort(key=lambda x: (x["type"], x["sample"]))
|
||||
|
||||
out_csv_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with out_csv_path.open("w", newline="") as f:
|
||||
w = csv.DictWriter(f, fieldnames=list(out_rows[0].keys()))
|
||||
w.writeheader()
|
||||
w.writerows(out_rows)
|
||||
|
||||
lines = []
|
||||
lines.append("# Stage2 ours vs routinator (decode+profile, selected_der_v2)\n")
|
||||
lines.append(f"- ours_csv: `{ours_path}`\n")
|
||||
lines.append(f"- rout_csv: `{rout_path}`\n")
|
||||
lines.append("\n")
|
||||
lines.append("| type | sample | size_bytes | complexity | ours ns/op | rout ns/op | ratio |\n")
|
||||
lines.append("|---|---|---:|---:|---:|---:|---:|\n")
|
||||
for r in out_rows:
|
||||
lines.append(
|
||||
f"| {r['type']} | {r['sample']} | {r['size_bytes']} | {r['complexity']} | "
|
||||
f"{r['ours_avg_ns_per_op']} | {r['rout_avg_ns_per_op']} | {r['ratio_ours_over_rout']} |\n"
|
||||
)
|
||||
|
||||
out_md_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_md_path.write_text("".join(lines), encoding="utf-8")
|
||||
PY
|
||||
|
||||
echo "Done." >&2
|
||||
echo "- routinator CSV: $ROUT_CSV" >&2
|
||||
echo "- compare CSV: $COMPARE_CSV" >&2
|
||||
echo "- compare MD: $COMPARE_MD" >&2
|
||||
Reference in New Issue
Block a user