20260421 优化内存布局，降低内存峰值需求压低到4.5GB左右，在4c8g 机器上，稳定跑snapshot + delta
This commit is contained in:
@@ -7,6 +7,7 @@ Usage:
|
||||
./scripts/compare/run_perf_compare_quick_remote.sh \
|
||||
--run-root <path> \
|
||||
--remote-root <path> \
|
||||
[--rir-set <mixed2|all5>] \
|
||||
[--ssh-target <user@host>] \
|
||||
[--rpki-client-bin <path>] \
|
||||
[--libtls-path <path>] \
|
||||
@@ -23,6 +24,7 @@ SSH_TARGET="${SSH_TARGET:-root@47.251.56.108}"
|
||||
RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
|
||||
LIBTLS_PATH="${LIBTLS_PATH:-/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0}"
|
||||
RP_RUN_MODE="${RP_RUN_MODE:-serial}"
|
||||
RIR_SET="${RIR_SET:-mixed2}"
|
||||
OURS_EXTRA_ARGS="${OURS_EXTRA_ARGS:-}"
|
||||
DRY_RUN=0
|
||||
|
||||
@@ -30,6 +32,7 @@ while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--run-root) RUN_ROOT="$2"; shift 2 ;;
|
||||
--remote-root) REMOTE_ROOT="$2"; shift 2 ;;
|
||||
--rir-set) RIR_SET="$2"; shift 2 ;;
|
||||
--ssh-target) SSH_TARGET="$2"; shift 2 ;;
|
||||
--rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
|
||||
--libtls-path) LIBTLS_PATH="$2"; shift 2 ;;
|
||||
@@ -43,6 +46,7 @@ done
|
||||
|
||||
[[ -n "$RUN_ROOT" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
|
||||
[[ "$RP_RUN_MODE" == "serial" || "$RP_RUN_MODE" == "parallel" ]] || { echo "invalid --rp-run-mode: $RP_RUN_MODE" >&2; usage; exit 2; }
|
||||
[[ "$RIR_SET" == "mixed2" || "$RIR_SET" == "all5" ]] || { echo "invalid --rir-set: $RIR_SET" >&2; usage; exit 2; }
|
||||
[[ "$DRY_RUN" -eq 1 || -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
|
||||
[[ "$DRY_RUN" -eq 1 || -f "$LIBTLS_PATH" ]] || { echo "libtls not found: $LIBTLS_PATH" >&2; exit 2; }
|
||||
|
||||
@@ -56,15 +60,50 @@ PY
|
||||
mkdir -p "$RUN_ROOT/steps/step-001/ours" "$RUN_ROOT/steps/step-001/rpki-client" "$RUN_ROOT/steps/step-001/compare"
|
||||
mkdir -p "$RUN_ROOT/steps/step-002/ours" "$RUN_ROOT/steps/step-002/rpki-client" "$RUN_ROOT/steps/step-002/compare"
|
||||
|
||||
APNIC_TAL="$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal"
|
||||
APNIC_TA="$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer"
|
||||
ARIN_TAL="$ROOT_DIR/tests/fixtures/tal/arin.tal"
|
||||
ARIN_TA="$ROOT_DIR/tests/fixtures/ta/arin-ta.cer"
|
||||
tal_path_for_rir() {
|
||||
case "$1" in
|
||||
afrinic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/afrinic.tal" ;;
|
||||
apnic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal" ;;
|
||||
arin) printf '%s' "$ROOT_DIR/tests/fixtures/tal/arin.tal" ;;
|
||||
lacnic) printf '%s' "$ROOT_DIR/tests/fixtures/tal/lacnic.tal" ;;
|
||||
ripe) printf '%s' "$ROOT_DIR/tests/fixtures/tal/ripe-ncc.tal" ;;
|
||||
*) echo "unknown rir: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
ta_path_for_rir() {
|
||||
case "$1" in
|
||||
afrinic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/afrinic-ta.cer" ;;
|
||||
apnic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer" ;;
|
||||
arin) printf '%s' "$ROOT_DIR/tests/fixtures/ta/arin-ta.cer" ;;
|
||||
lacnic) printf '%s' "$ROOT_DIR/tests/fixtures/ta/lacnic-ta.cer" ;;
|
||||
ripe) printf '%s' "$ROOT_DIR/tests/fixtures/ta/ripe-ncc-ta.cer" ;;
|
||||
*) echo "unknown rir: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
case "$RIR_SET" in
|
||||
mixed2)
|
||||
RIRS=(apnic arin)
|
||||
SCOPE_LABEL="APNIC+ARIN mixed release two-step synchronized compare"
|
||||
;;
|
||||
all5)
|
||||
RIRS=(afrinic apnic arin lacnic ripe)
|
||||
SCOPE_LABEL="all-five-RIR mixed release two-step synchronized compare"
|
||||
;;
|
||||
esac
|
||||
|
||||
COPY_FILES=()
|
||||
for rir in "${RIRS[@]}"; do
|
||||
COPY_FILES+=("$(tal_path_for_rir "$rir")" "$(ta_path_for_rir "$rir")")
|
||||
done
|
||||
|
||||
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||
cat <<EOF2
|
||||
workflow_name=性能对比测试快速版
|
||||
scope=APNIC+ARIN mixed release two-step synchronized compare
|
||||
scope=$SCOPE_LABEL
|
||||
rir_set=$RIR_SET
|
||||
rirs=${RIRS[*]}
|
||||
run_root=$RUN_ROOT
|
||||
remote_root=$REMOTE_ROOT
|
||||
ssh_target=$SSH_TARGET
|
||||
@@ -87,31 +126,72 @@ trap cleanup_remote EXIT
|
||||
)
|
||||
|
||||
ssh "$SSH_TARGET" "set -e; id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; rm -rf '$REMOTE_ROOT'; mkdir -p '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client' '$REMOTE_ROOT/steps/step-001/ours' '$REMOTE_ROOT/steps/step-001/rpki-client' '$REMOTE_ROOT/steps/step-002/ours' '$REMOTE_ROOT/steps/step-002/rpki-client'"
|
||||
scp "$ROOT_DIR/target/release/rpki" "$APNIC_TAL" "$APNIC_TA" "$ARIN_TAL" "$ARIN_TA" "$SSH_TARGET:$REMOTE_ROOT/"
|
||||
scp "$ROOT_DIR/target/release/rpki" "${COPY_FILES[@]}" "$SSH_TARGET:$REMOTE_ROOT/"
|
||||
scp "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
|
||||
scp "$LIBTLS_PATH" "$SSH_TARGET:$REMOTE_ROOT/lib/libtls.so.28"
|
||||
printf '%s' "$OURS_EXTRA_ARGS" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/ours-extra-args.txt'"
|
||||
printf '%s' "$RP_RUN_MODE" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/rp-run-mode.txt'"
|
||||
printf '%s' "$RIR_SET" | ssh "$SSH_TARGET" "cat > '$REMOTE_ROOT/rir-set.txt'"
|
||||
|
||||
run_step() {
|
||||
local step_id="$1"
|
||||
local kind="$2"
|
||||
local local_step="$RUN_ROOT/steps/$step_id"
|
||||
|
||||
ssh "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$step_id" "$kind" "$OURS_EXTRA_ARGS" "$RP_RUN_MODE" <<'EOS'
|
||||
ssh "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$step_id" "$kind" <<'EOS'
|
||||
set -euo pipefail
|
||||
REMOTE_ROOT="$1"
|
||||
STEP_ID="$2"
|
||||
KIND="$3"
|
||||
OURS_EXTRA_ARGS="$4"
|
||||
RP_RUN_MODE="$5"
|
||||
|
||||
cd "$REMOTE_ROOT"
|
||||
mkdir -p "steps/$STEP_ID/ours" "steps/$STEP_ID/rpki-client"
|
||||
OURS_EXTRA_ARGS="$(cat ours-extra-args.txt)"
|
||||
RP_RUN_MODE="$(cat rp-run-mode.txt)"
|
||||
RIR_SET="$(cat rir-set.txt)"
|
||||
OURS_EXTRA_ARGV=()
|
||||
if [[ -n "$OURS_EXTRA_ARGS" ]]; then
|
||||
# shellcheck disable=SC2206
|
||||
OURS_EXTRA_ARGV=($OURS_EXTRA_ARGS)
|
||||
fi
|
||||
|
||||
case "$RIR_SET" in
|
||||
mixed2) RIRS=(apnic arin) ;;
|
||||
all5) RIRS=(afrinic apnic arin lacnic ripe) ;;
|
||||
*) echo "invalid rir set: $RIR_SET" >&2; exit 2 ;;
|
||||
esac
|
||||
|
||||
tal_file_for_rir() {
|
||||
case "$1" in
|
||||
afrinic) printf '%s' "afrinic.tal" ;;
|
||||
apnic) printf '%s' "apnic-rfc7730-https.tal" ;;
|
||||
arin) printf '%s' "arin.tal" ;;
|
||||
lacnic) printf '%s' "lacnic.tal" ;;
|
||||
ripe) printf '%s' "ripe-ncc.tal" ;;
|
||||
*) echo "unknown rir: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
ta_file_for_rir() {
|
||||
case "$1" in
|
||||
afrinic) printf '%s' "afrinic-ta.cer" ;;
|
||||
apnic) printf '%s' "apnic-ta.cer" ;;
|
||||
arin) printf '%s' "arin-ta.cer" ;;
|
||||
lacnic) printf '%s' "lacnic-ta.cer" ;;
|
||||
ripe) printf '%s' "ripe-ncc-ta.cer" ;;
|
||||
*) echo "unknown rir: $1" >&2; exit 2 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
OURS_TAL_ARGS=()
|
||||
CLIENT_TAL_ARGS=()
|
||||
for rir in "${RIRS[@]}"; do
|
||||
tal_file="$(tal_file_for_rir "$rir")"
|
||||
ta_file="$(ta_file_for_rir "$rir")"
|
||||
OURS_TAL_ARGS+=(--tal-path "$tal_file" --ta-path "$ta_file")
|
||||
CLIENT_TAL_ARGS+=(-t "../../$tal_file")
|
||||
done
|
||||
|
||||
if [[ "$KIND" == "snapshot" ]]; then
|
||||
rm -rf state/ours/work-db state/ours/raw-store.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
|
||||
fi
|
||||
@@ -142,8 +222,7 @@ PY
|
||||
env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=0 ./rpki \
|
||||
--db state/ours/work-db \
|
||||
--raw-store-db state/ours/raw-store.db \
|
||||
--tal-path apnic-rfc7730-https.tal --ta-path apnic-ta.cer \
|
||||
--tal-path arin.tal --ta-path arin-ta.cer \
|
||||
"${OURS_TAL_ARGS[@]}" \
|
||||
--parallel-phase1 \
|
||||
"${OURS_EXTRA_ARGV[@]}" \
|
||||
--ccr-out "steps/$STEP_ID/ours/result.ccr" \
|
||||
@@ -189,8 +268,7 @@ PY
|
||||
set +e
|
||||
LD_LIBRARY_PATH="$REMOTE_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" "$REMOTE_ROOT/bin/rpki-client" \
|
||||
-vv \
|
||||
-t ../../apnic-rfc7730-https.tal \
|
||||
-t ../../arin.tal \
|
||||
"${CLIENT_TAL_ARGS[@]}" \
|
||||
-d cache out \
|
||||
> "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/run.log" 2>&1
|
||||
exit_code=$?
|
||||
@@ -286,14 +364,16 @@ PY
|
||||
run_step step-001 snapshot
|
||||
run_step step-002 delta
|
||||
|
||||
python3 - <<'PY' "$RUN_ROOT/steps/step-001/step-summary.json" "$RUN_ROOT/steps/step-002/step-summary.json" "$RUN_ROOT/summary.json" "$RP_RUN_MODE" "$OURS_EXTRA_ARGS"
|
||||
python3 - <<'PY' "$RUN_ROOT/steps/step-001/step-summary.json" "$RUN_ROOT/steps/step-002/step-summary.json" "$RUN_ROOT/summary.json" "$RP_RUN_MODE" "$OURS_EXTRA_ARGS" "$RIR_SET" "$SCOPE_LABEL" "${RIRS[@]}"
|
||||
import json, sys
|
||||
steps = [json.load(open(p)) for p in sys.argv[1:3]]
|
||||
summary = {
|
||||
"workflowName": "性能对比测试快速版",
|
||||
"scope": "APNIC+ARIN mixed release two-step synchronized compare",
|
||||
"scope": sys.argv[7],
|
||||
"rpRunMode": sys.argv[4],
|
||||
"oursExtraArgs": sys.argv[5],
|
||||
"rirSet": sys.argv[6],
|
||||
"rirs": sys.argv[8:],
|
||||
"steps": steps,
|
||||
}
|
||||
json.dump(summary, open(sys.argv[3], "w"), indent=2, ensure_ascii=False)
|
||||
|
||||
Reference in New Issue
Block a user