manifest decode & profile validate optimization
This commit is contained in:
195
scripts/manifest_perf_compare_m2.sh
Executable file
195
scripts/manifest_perf_compare_m2.sh
Executable file
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# M2: Run per-sample decode+profile benchmark (Ours vs Routinator) on selected_der fixtures.
|
||||
#
|
||||
# Outputs:
|
||||
# - specs/develop/20260224/data/m2_manifest_decode_profile_compare.csv
|
||||
# - specs/develop/20260224/data/m2_raw.log
|
||||
#
|
||||
# Note: This script assumes Routinator benchmark repo exists at:
|
||||
# /home/yuyr/dev/rust_playground/routinator/benchmark
|
||||
#
|
||||
# It also assumes fixtures exist under:
|
||||
# rpki/tests/benchmark/selected_der/*.mft
|
||||
# routinator/benchmark/fixtures/selected_der/*.mft
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
RPKI_DIR="$ROOT_DIR"
|
||||
OURS_BENCH_DIR="$RPKI_DIR/benchmark/ours_manifest_bench"
|
||||
|
||||
ROUT_BENCH_DIR="${ROUT_BENCH_DIR:-/home/yuyr/dev/rust_playground/routinator/benchmark}"
|
||||
ROUT_BIN="$ROUT_BENCH_DIR/target/release/routinator-manifest-benchmark"
|
||||
|
||||
DATE_TAG="${DATE_TAG:-20260224}"
|
||||
OUT_DIR="$RPKI_DIR/../specs/develop/${DATE_TAG}/data"
|
||||
OUT_CSV="${OUT_CSV:-$OUT_DIR/m2_manifest_decode_profile_compare.csv}"
|
||||
OUT_RAW="${OUT_RAW:-$OUT_DIR/m2_raw.log}"
|
||||
|
||||
REPEATS="${REPEATS:-3}"
|
||||
|
||||
# Iterations / warmups (kept moderate for interactive iteration).
|
||||
ITER_SMALL="${ITER_SMALL:-20000}"
|
||||
ITER_MEDIUM="${ITER_MEDIUM:-20000}"
|
||||
ITER_LARGE="${ITER_LARGE:-20000}"
|
||||
ITER_XLARGE="${ITER_XLARGE:-2000}"
|
||||
|
||||
WARM_SMALL="${WARM_SMALL:-2000}"
|
||||
WARM_MEDIUM="${WARM_MEDIUM:-2000}"
|
||||
WARM_LARGE="${WARM_LARGE:-2000}"
|
||||
WARM_XLARGE="${WARM_XLARGE:-200}"
|
||||
|
||||
SAMPLES=(
|
||||
small-01
|
||||
small-02
|
||||
medium-01
|
||||
medium-02
|
||||
large-01
|
||||
large-02
|
||||
xlarge-01
|
||||
xlarge-02
|
||||
)
|
||||
|
||||
mkdir -p "$OUT_DIR"
|
||||
: > "$OUT_RAW"
|
||||
|
||||
echo "sample,bucket,manifest_file_count,ours_avg_ns_per_op,ours_ops_per_s,rout_avg_ns_per_op,rout_ops_per_s,ratio_ours_over_rout,iterations,repeats,warmup" > "$OUT_CSV"
|
||||
|
||||
echo "[1/3] Build ours benchmark (release)..." | tee -a "$OUT_RAW"
|
||||
(cd "$OURS_BENCH_DIR" && cargo build --release -q)
|
||||
OURS_BIN="$OURS_BENCH_DIR/target/release/ours-manifest-bench"
|
||||
|
||||
echo "[2/3] Build routinator benchmark (release)..." | tee -a "$OUT_RAW"
|
||||
(cd "$ROUT_BENCH_DIR" && cargo build --release -q)
|
||||
|
||||
taskset_prefix=""
|
||||
if command -v taskset >/dev/null 2>&1; then
|
||||
if [[ -n "${TASKSET_CPU:-}" ]]; then
|
||||
taskset_prefix="taskset -c ${TASKSET_CPU}"
|
||||
fi
|
||||
fi
|
||||
|
||||
bucket_for() {
|
||||
local s="$1"
|
||||
case "$s" in
|
||||
small-*) echo "small" ;;
|
||||
medium-*) echo "medium" ;;
|
||||
large-*) echo "large" ;;
|
||||
xlarge-*) echo "xlarge" ;;
|
||||
*) echo "unknown" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
iters_for() {
|
||||
local b="$1"
|
||||
case "$b" in
|
||||
small) echo "$ITER_SMALL" ;;
|
||||
medium) echo "$ITER_MEDIUM" ;;
|
||||
large) echo "$ITER_LARGE" ;;
|
||||
xlarge) echo "$ITER_XLARGE" ;;
|
||||
*) echo "$ITER_MEDIUM" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
warm_for() {
|
||||
local b="$1"
|
||||
case "$b" in
|
||||
small) echo "$WARM_SMALL" ;;
|
||||
medium) echo "$WARM_MEDIUM" ;;
|
||||
large) echo "$WARM_LARGE" ;;
|
||||
xlarge) echo "$WARM_XLARGE" ;;
|
||||
*) echo "$WARM_MEDIUM" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
run_ours() {
|
||||
local sample="$1"
|
||||
local iters="$2"
|
||||
local warm="$3"
|
||||
local ours_fixture="$RPKI_DIR/tests/benchmark/selected_der/${sample}.mft"
|
||||
if [[ ! -f "$ours_fixture" ]]; then
|
||||
echo "ours fixture not found: $ours_fixture" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "### ours $sample" >> "$OUT_RAW"
|
||||
local out
|
||||
out=$($taskset_prefix "$OURS_BIN" --manifest "$ours_fixture" --iterations "$iters" --warmup-iterations "$warm" --repeats "$REPEATS")
|
||||
echo "$out" >> "$OUT_RAW"
|
||||
|
||||
local line
|
||||
line=$(echo "$out" | rg "^\\| ${sample} \\|" | tail -n 1)
|
||||
if [[ -z "${line:-}" ]]; then
|
||||
echo "failed to parse ours output for $sample" >&2
|
||||
exit 1
|
||||
fi
|
||||
# Expected final row: | sample | avg ns/op | ops/s | file count |
|
||||
local avg ops cnt
|
||||
avg=$(echo "$line" | awk -F'|' '{gsub(/^ +| +$/,"",$3); print $3}')
|
||||
ops=$(echo "$line" | awk -F'|' '{gsub(/^ +| +$/,"",$4); print $4}')
|
||||
cnt=$(echo "$line" | awk -F'|' '{gsub(/^ +| +$/,"",$5); print $5}')
|
||||
echo "$avg,$ops,$cnt"
|
||||
}
|
||||
|
||||
run_rout() {
|
||||
local sample="$1"
|
||||
local iters="$2"
|
||||
local warm="$3"
|
||||
local rout_fixture="$ROUT_BENCH_DIR/fixtures/selected_der/${sample}.mft"
|
||||
if [[ ! -f "$rout_fixture" ]]; then
|
||||
echo "routinator fixture not found: $rout_fixture" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "### routinator $sample" >> "$OUT_RAW"
|
||||
local out
|
||||
out=$(
|
||||
$taskset_prefix "$ROUT_BIN" \
|
||||
--target decode_only \
|
||||
--manifest "$rout_fixture" \
|
||||
--issuer "$ROUT_BENCH_DIR/fixtures/ta.cer" \
|
||||
--iterations "$iters" \
|
||||
--repeats "$REPEATS" \
|
||||
--warmup-iterations "$warm" \
|
||||
--strict false
|
||||
)
|
||||
echo "$out" >> "$OUT_RAW"
|
||||
|
||||
local avg_line cnt_line
|
||||
avg_line=$(echo "$out" | rg "^ avg:")
|
||||
cnt_line=$(echo "$out" | rg "^ manifest_file_count:")
|
||||
|
||||
local avg_ns ops_s cnt
|
||||
avg_ns=$(echo "$avg_line" | awk '{print $2}')
|
||||
ops_s=$(echo "$avg_line" | awk '{gsub(/[()]/,"",$4); print $4}')
|
||||
cnt=$(echo "$cnt_line" | awk '{print $2}')
|
||||
echo "$avg_ns,$ops_s,$cnt"
|
||||
}
|
||||
|
||||
echo "[3/3] Run per-sample benchmarks..." | tee -a "$OUT_RAW"
|
||||
for s in "${SAMPLES[@]}"; do
|
||||
b=$(bucket_for "$s")
|
||||
it=$(iters_for "$b")
|
||||
warm=$(warm_for "$b")
|
||||
|
||||
IFS=, read -r ours_avg ours_ops ours_cnt < <(run_ours "$s" "$it" "$warm")
|
||||
IFS=, read -r rout_avg rout_ops rout_cnt < <(run_rout "$s" "$it" "$warm")
|
||||
|
||||
if [[ "$ours_cnt" != "$rout_cnt" ]]; then
|
||||
echo "WARNING: file count differs for $s (ours=$ours_cnt rout=$rout_cnt)" | tee -a "$OUT_RAW"
|
||||
fi
|
||||
|
||||
ratio=$(python3 - <<PY
|
||||
o=float("$ours_avg")
|
||||
r=float("$rout_avg")
|
||||
print(f"{(o/r):.4f}" if r != 0 else "inf")
|
||||
PY
|
||||
)
|
||||
|
||||
echo "$s,$b,$ours_cnt,$ours_avg,$ours_ops,$rout_avg,$rout_ops,$ratio,$it,$REPEATS,$warm" >> "$OUT_CSV"
|
||||
echo >> "$OUT_RAW"
|
||||
done
|
||||
|
||||
echo "Done."
|
||||
echo "- CSV: $OUT_CSV"
|
||||
echo "- Raw: $OUT_RAW"
|
||||
142
scripts/manifest_perf_profile_m3.sh
Executable file
142
scripts/manifest_perf_profile_m3.sh
Executable file
@@ -0,0 +1,142 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# M3: Generate flamegraphs + top hotspots for Manifest decode+profile (Ours vs Routinator).
|
||||
#
|
||||
# Outputs under:
|
||||
# specs/develop/20260224/flamegraph/
|
||||
# specs/develop/20260224/hotspots/
|
||||
# specs/develop/20260224/perf/
|
||||
#
|
||||
# Notes:
|
||||
# - On WSL2, /usr/bin/perf is often a wrapper that fails. This script uses a real perf binary
|
||||
# from /usr/lib/linux-tools/*/perf (if present).
|
||||
# - Ours profiling uses perf + flamegraph --perfdata to avoid rebuilding the whole crate graph
|
||||
# with RocksDB.
|
||||
|
||||
ROOT_REPO="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
||||
RPKI_DIR="$ROOT_REPO/rpki"
|
||||
|
||||
DATE_TAG="${DATE_TAG:-20260224}"
|
||||
OUT_BASE="$ROOT_REPO/specs/develop/${DATE_TAG}"
|
||||
OUT_FLAME="$OUT_BASE/flamegraph"
|
||||
OUT_HOT="$OUT_BASE/hotspots"
|
||||
OUT_PERF="$OUT_BASE/perf"
|
||||
|
||||
RUN_TAG="${RUN_TAG:-p2}"
|
||||
|
||||
OURS_BENCH_DIR="$RPKI_DIR/benchmark/ours_manifest_bench"
|
||||
OURS_BIN="$OURS_BENCH_DIR/target/release/ours-manifest-bench"
|
||||
|
||||
ROUT_BENCH_DIR="${ROUT_BENCH_DIR:-/home/yuyr/dev/rust_playground/routinator/benchmark}"
|
||||
ROUT_BIN="$ROUT_BENCH_DIR/target/release/routinator-manifest-benchmark"
|
||||
ROUT_ISSUER="$ROUT_BENCH_DIR/fixtures/ta.cer"
|
||||
|
||||
PROFILE_HZ="${PROFILE_HZ:-99}"
|
||||
|
||||
mkdir -p "$OUT_FLAME" "$OUT_HOT" "$OUT_PERF"
|
||||
|
||||
PERF_WRAPPER_OUT="$(perf --version 2>&1 || true)"
|
||||
PERF_REAL=""
|
||||
if echo "${PERF_WRAPPER_OUT}" | grep -q "WARNING: perf not found for kernel"; then
|
||||
PERF_REAL="$(ls -1 /usr/lib/linux-tools/*/perf 2>/dev/null | head -n 1 || true)"
|
||||
else
|
||||
PERF_REAL="$(command -v perf || true)"
|
||||
fi
|
||||
|
||||
if [[ -z "${PERF_REAL}" ]]; then
|
||||
echo "ERROR: usable perf binary not found (wrapper detected and no /usr/lib/linux-tools/*/perf)." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
SHIM_DIR="$RPKI_DIR/target/bench/tools"
|
||||
mkdir -p "$SHIM_DIR"
|
||||
cat > "$SHIM_DIR/perf" <<EOF
|
||||
#!/usr/bin/env bash
|
||||
exec "${PERF_REAL}" "\$@"
|
||||
EOF
|
||||
chmod +x "$SHIM_DIR/perf"
|
||||
export PATH="$SHIM_DIR:$PATH"
|
||||
|
||||
echo "Using perf: $PERF_REAL"
|
||||
|
||||
echo "[1/3] Build ours benchmark with frame pointers..."
|
||||
(cd "$OURS_BENCH_DIR" && RUSTFLAGS="-C force-frame-pointers=yes" cargo build --release -q)
|
||||
|
||||
echo "[2/3] Build routinator benchmark (release)..."
|
||||
(cd "$ROUT_BENCH_DIR" && cargo build --release -q)
|
||||
|
||||
taskset_prefix=""
|
||||
if command -v taskset >/dev/null 2>&1; then
|
||||
taskset_prefix="taskset -c 0"
|
||||
fi
|
||||
|
||||
profile_ours() {
|
||||
local sample="$1"
|
||||
local iters="$2"
|
||||
local warm="$3"
|
||||
local fixture="$RPKI_DIR/tests/benchmark/selected_der/${sample}.mft"
|
||||
if [[ ! -f "$fixture" ]]; then
|
||||
echo "ERROR: ours fixture not found: $fixture" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local perfdata="$OUT_PERF/ours_${sample}_${RUN_TAG}.perf.data"
|
||||
local svg="$OUT_FLAME/ours_${sample}_${RUN_TAG}.svg"
|
||||
local tsv="$OUT_HOT/ours_${sample}_${RUN_TAG}.tsv"
|
||||
|
||||
echo "== ours $sample (iters=$iters warmup=$warm hz=$PROFILE_HZ)"
|
||||
$taskset_prefix perf record -o "$perfdata" -F "$PROFILE_HZ" -g -- \
|
||||
"$OURS_BIN" --manifest "$fixture" --iterations "$iters" --warmup-iterations "$warm" --repeats 1 >/dev/null
|
||||
|
||||
flamegraph --perfdata "$perfdata" --output "$svg" --title "ours ${sample} ManifestObject::decode_der" --deterministic >/dev/null
|
||||
|
||||
perf report -i "$perfdata" --stdio --no-children --sort symbol --percent-limit 0.5 \
|
||||
| awk '/^[[:space:]]*[0-9.]+%/ {pct=$1; sub(/%/,"",pct); $1=""; sub(/^[[:space:]]+/,""); print pct "\t" $0}' \
|
||||
> "$tsv"
|
||||
}
|
||||
|
||||
profile_routinator() {
|
||||
local sample="$1"
|
||||
local iters="$2"
|
||||
local warm="$3"
|
||||
local fixture="$ROUT_BENCH_DIR/fixtures/selected_der/${sample}.mft"
|
||||
if [[ ! -f "$fixture" ]]; then
|
||||
echo "ERROR: routinator fixture not found: $fixture" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local svg="$OUT_FLAME/routinator_${sample}_${RUN_TAG}.svg"
|
||||
local tsv="$OUT_HOT/routinator_${sample}_${RUN_TAG}.tsv"
|
||||
|
||||
echo "== routinator $sample (iters=$iters warmup=$warm hz=$PROFILE_HZ)"
|
||||
$taskset_prefix "$ROUT_BIN" \
|
||||
--target decode_only \
|
||||
--manifest "$fixture" \
|
||||
--issuer "$ROUT_ISSUER" \
|
||||
--iterations "$iters" \
|
||||
--repeats 1 \
|
||||
--warmup-iterations "$warm" \
|
||||
--strict false \
|
||||
--profile-hz "$PROFILE_HZ" \
|
||||
--flamegraph "$svg" \
|
||||
--hotspots "$tsv" \
|
||||
>/dev/null
|
||||
}
|
||||
|
||||
echo "[3/3] Profile samples..."
|
||||
|
||||
# Choose iterations so each capture runs ~10-20s serially.
|
||||
profile_ours small-01 200000 0
|
||||
profile_routinator small-01 200000 0
|
||||
|
||||
profile_ours large-02 50000 0
|
||||
profile_routinator large-02 50000 0
|
||||
|
||||
profile_ours xlarge-02 5000 0
|
||||
profile_routinator xlarge-02 5000 0
|
||||
|
||||
echo "Done."
|
||||
echo "- Flamegraphs: $OUT_FLAME/"
|
||||
echo "- Hotspots: $OUT_HOT/"
|
||||
echo "- Perf data: $OUT_PERF/"
|
||||
Reference in New Issue
Block a user