20260418 优化去掉冗余存储repo object，mix quick最快170s

2026-04-18 14:10:47 +08:00
parent f485786470
commit 417c82bef6
34 changed files with 1482 additions and 328 deletions
--- a/scripts/compare/run_perf_compare_quick_remote.sh
+++ b/scripts/compare/run_perf_compare_quick_remote.sh
@@ -0,0 +1,263 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  ./scripts/compare/run_perf_compare_quick_remote.sh \
+    --run-root <path> \
+    --remote-root <path> \
+    [--ssh-target <user@host>] \
+    [--rpki-client-bin <path>] \
+    [--libtls-path <path>] \
+    [--dry-run]
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+RUN_ROOT=""
+REMOTE_ROOT=""
+SSH_TARGET="${SSH_TARGET:-root@47.251.56.108}"
+RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
+LIBTLS_PATH="${LIBTLS_PATH:-/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28.0.0}"
+DRY_RUN=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --run-root) RUN_ROOT="$2"; shift 2 ;;
+    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
+    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --libtls-path) LIBTLS_PATH="$2"; shift 2 ;;
+    --dry-run) DRY_RUN=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage; exit 2 ;;
+  esac
+done
+
+[[ -n "$RUN_ROOT" && -n "$REMOTE_ROOT" ]] || { usage >&2; exit 2; }
+[[ "$DRY_RUN" -eq 1 || -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+[[ "$DRY_RUN" -eq 1 || -f "$LIBTLS_PATH" ]] || { echo "libtls not found: $LIBTLS_PATH" >&2; exit 2; }
+
+RUN_ROOT="$(python3 - <<'PY' "$RUN_ROOT"
+from pathlib import Path
+import sys
+print(Path(sys.argv[1]).resolve())
+PY
+)"
+
+mkdir -p "$RUN_ROOT/steps/step-001/ours" "$RUN_ROOT/steps/step-001/rpki-client" "$RUN_ROOT/steps/step-001/compare"
+mkdir -p "$RUN_ROOT/steps/step-002/ours" "$RUN_ROOT/steps/step-002/rpki-client" "$RUN_ROOT/steps/step-002/compare"
+
+APNIC_TAL="$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal"
+APNIC_TA="$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer"
+ARIN_TAL="$ROOT_DIR/tests/fixtures/tal/arin.tal"
+ARIN_TA="$ROOT_DIR/tests/fixtures/ta/arin-ta.cer"
+
+if [[ "$DRY_RUN" -eq 1 ]]; then
+  cat <<EOF2
+workflow_name=性能对比测试快速版
+scope=APNIC+ARIN mixed release two-step synchronized compare
+run_root=$RUN_ROOT
+remote_root=$REMOTE_ROOT
+ssh_target=$SSH_TARGET
+EOF2
+  exit 0
+fi
+
+if [[ ! -x "$ROOT_DIR/target/release/rpki" || ! -x "$ROOT_DIR/target/release/ccr_to_compare_views" ]]; then
+  (
+    cd "$ROOT_DIR"
+    cargo build --release --bin rpki --bin ccr_to_compare_views
+  )
+fi
+
+ssh "$SSH_TARGET" "set -e; id -u _rpki-client >/dev/null 2>&1 || useradd -r -M -s /usr/sbin/nologin _rpki-client || true; rm -rf '$REMOTE_ROOT'; mkdir -p '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client' '$REMOTE_ROOT/steps/step-001/ours' '$REMOTE_ROOT/steps/step-001/rpki-client' '$REMOTE_ROOT/steps/step-002/ours' '$REMOTE_ROOT/steps/step-002/rpki-client'"
+scp "$ROOT_DIR/target/release/rpki" "$APNIC_TAL" "$APNIC_TA" "$ARIN_TAL" "$ARIN_TA" "$SSH_TARGET:$REMOTE_ROOT/"
+scp "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
+scp "$LIBTLS_PATH" "$SSH_TARGET:$REMOTE_ROOT/lib/libtls.so.28"
+
+run_step() {
+  local step_id="$1"
+  local kind="$2"
+  local local_step="$RUN_ROOT/steps/$step_id"
+
+  ssh "$SSH_TARGET" bash -s -- "$REMOTE_ROOT" "$step_id" "$kind" <<'EOS'
+set -euo pipefail
+REMOTE_ROOT="$1"
+STEP_ID="$2"
+KIND="$3"
+
+cd "$REMOTE_ROOT"
+mkdir -p "steps/$STEP_ID/ours" "steps/$STEP_ID/rpki-client"
+
+if [[ "$KIND" == "snapshot" ]]; then
+  rm -rf state/ours/work-db state/ours/raw-store.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
+fi
+mkdir -p state/ours/work-db state/ours/raw-store.db state/rpki-client/cache state/rpki-client/out state/rpki-client/ta state/rpki-client/.ta
+chmod 0777 state/ours/work-db state/ours/raw-store.db
+chmod -R 0777 state/rpki-client
+
+START_EPOCH="$(python3 - <<'PY'
+import time
+print(time.time() + 3.0)
+PY
+)"
+
+(
+  python3 - <<'PY' "$START_EPOCH"
+import sys, time
+x = float(sys.argv[1])
+d = x - time.time()
+if d > 0:
+    time.sleep(d)
+PY
+  started_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  set +e
+  env RPKI_PROGRESS_LOG=1 RPKI_PROGRESS_SLOW_SECS=0 ./rpki \
+    --db state/ours/work-db \
+    --raw-store-db state/ours/raw-store.db \
+    --tal-path apnic-rfc7730-https.tal --ta-path apnic-ta.cer \
+    --tal-path arin.tal --ta-path arin-ta.cer \
+    --parallel-phase1 \
+    --ccr-out "steps/$STEP_ID/ours/result.ccr" \
+    --report-json "steps/$STEP_ID/ours/report.json" \
+    > "steps/$STEP_ID/ours/run.log" 2>&1
+  exit_code=$?
+  set -e
+  finished_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  python3 - <<'PY' "steps/$STEP_ID/ours/round-result.json" "$STEP_ID" "$KIND" "$started_ms" "$finished_ms" "$exit_code"
+import json, sys
+path, step_id, kind, started_ms, finished_ms, exit_code = sys.argv[1:]
+json.dump(
+    {
+        "stepId": step_id,
+        "kind": kind,
+        "durationMs": int(finished_ms) - int(started_ms),
+        "exitCode": int(exit_code),
+    },
+    open(path, "w"),
+    indent=2,
+)
+PY
+) &
+OURS_PID=$!
+
+(
+  cd state/rpki-client
+  python3 - <<'PY' "$START_EPOCH"
+import sys, time
+x = float(sys.argv[1])
+d = x - time.time()
+if d > 0:
+    time.sleep(d)
+PY
+  started_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  set +e
+  LD_LIBRARY_PATH="$REMOTE_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" "$REMOTE_ROOT/bin/rpki-client" \
+    -vv \
+    -t ../../apnic-rfc7730-https.tal \
+    -t ../../arin.tal \
+    -d cache out \
+    > "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/run.log" 2>&1
+  exit_code=$?
+  set -e
+  cp out/rpki.ccr "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/result.ccr" 2>/dev/null || true
+  cp out/openbgpd "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/openbgpd" 2>/dev/null || true
+  finished_ms="$(python3 - <<'PY'
+import time
+print(int(time.time() * 1000))
+PY
+)"
+  python3 - <<'PY' "$REMOTE_ROOT/steps/$STEP_ID/rpki-client/round-result.json" "$STEP_ID" "$KIND" "$started_ms" "$finished_ms" "$exit_code"
+import json, sys
+path, step_id, kind, started_ms, finished_ms, exit_code = sys.argv[1:]
+json.dump(
+    {
+        "stepId": step_id,
+        "kind": kind,
+        "durationMs": int(finished_ms) - int(started_ms),
+        "exitCode": int(exit_code),
+    },
+    open(path, "w"),
+    indent=2,
+)
+PY
+) &
+CLIENT_PID=$!
+
+wait "$OURS_PID"
+wait "$CLIENT_PID"
+EOS
+
+  for rel in result.ccr round-result.json run.log stage-timing.json; do
+    scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/ours/$rel" "$local_step/ours/"
+  done
+  for rel in result.ccr round-result.json run.log openbgpd; do
+    scp -C "$SSH_TARGET:$REMOTE_ROOT/steps/$step_id/rpki-client/$rel" "$local_step/rpki-client/" || true
+  done
+
+  "$ROOT_DIR/scripts/periodic/compare_ccr_round.sh" \
+    --ours-ccr "$local_step/ours/result.ccr" \
+    --rpki-client-ccr "$local_step/rpki-client/result.ccr" \
+    --out-dir "$local_step/compare" \
+    --trust-anchor unknown >/dev/null
+
+  python3 - <<'PY' "$local_step/ours/round-result.json" "$local_step/rpki-client/round-result.json" "$local_step/ours/stage-timing.json" "$local_step/compare/compare-summary.json" "$local_step/step-summary.json"
+import json, sys
+ours = json.load(open(sys.argv[1]))
+client = json.load(open(sys.argv[2]))
+stage = json.load(open(sys.argv[3]))
+compare = json.load(open(sys.argv[4]))
+json.dump(
+    {
+        "stepId": ours["stepId"],
+        "kind": ours["kind"],
+        "oursDurationMs": ours["durationMs"],
+        "rpkiClientDurationMs": client["durationMs"],
+        "oursExitCode": ours["exitCode"],
+        "rpkiClientExitCode": client["exitCode"],
+        "oursTotalMs": stage["total_ms"],
+        "oursRepoSyncMsTotal": stage["repo_sync_ms_total"],
+        "oursVrps": compare["vrps"]["ours"],
+        "rpkiClientVrps": compare["vrps"]["rpkiClient"],
+        "oursVaps": compare["vaps"]["ours"],
+        "rpkiClientVaps": compare["vaps"]["rpkiClient"],
+        "vrpMatch": compare["vrps"]["match"],
+        "vapMatch": compare["vaps"]["match"],
+        "allMatch": compare["allMatch"],
+        "onlyInOurs": len(compare["vrps"]["onlyInOurs"]),
+        "onlyInRpkiClient": len(compare["vrps"]["onlyInRpkiClient"]),
+    },
+    open(sys.argv[5], "w"),
+    indent=2,
+)
+PY
+}
+
+run_step step-001 snapshot
+run_step step-002 delta
+
+python3 - <<'PY' "$RUN_ROOT/steps/step-001/step-summary.json" "$RUN_ROOT/steps/step-002/step-summary.json" "$RUN_ROOT/summary.json"
+import json, sys
+steps = [json.load(open(p)) for p in sys.argv[1:3]]
+summary = {
+    "workflowName": "性能对比测试快速版",
+    "scope": "APNIC+ARIN mixed release two-step synchronized compare",
+    "steps": steps,
+}
+json.dump(summary, open(sys.argv[3], "w"), indent=2, ensure_ascii=False)
+print(json.dumps(summary, indent=2, ensure_ascii=False))
+PY
--- a/scripts/periodic/run_apnic_parallel_dual_rp_periodic_ccr_compare.sh
+++ b/scripts/periodic/run_apnic_parallel_dual_rp_periodic_ccr_compare.sh
@@ -21,6 +21,7 @@ RUN_ROOT=""
 SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
 REMOTE_ROOT=""
 RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
+RPKI_CLIENT_LIBTLS_PATH="${RPKI_CLIENT_LIBTLS_PATH:-/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28}"
 ROUND_COUNT=10
 INTERVAL_SECS=600
 START_AT=""
@@ -32,6 +33,7 @@ while [[ $# -gt 0 ]]; do
    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --rpki-client-libtls) RPKI_CLIENT_LIBTLS_PATH="$2"; shift 2 ;;
    --round-count) ROUND_COUNT="$2"; shift 2 ;;
    --interval-secs) INTERVAL_SECS="$2"; shift 2 ;;
    --start-at) START_AT="$2"; shift 2 ;;
@@ -47,6 +49,7 @@ done
 if [[ "$DRY_RUN" -ne 1 ]]; then
  [[ -n "$REMOTE_ROOT" ]] || { echo "--remote-root is required unless --dry-run" >&2; exit 2; }
  [[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+  [[ -f "$RPKI_CLIENT_LIBTLS_PATH" ]] || { echo "rpki-client libtls not found: $RPKI_CLIENT_LIBTLS_PATH" >&2; exit 2; }
 fi

 mkdir -p "$RUN_ROOT"
@@ -143,9 +146,10 @@ rsync -a --delete \
  --exclude target \
  --exclude .git \
  "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/repo/"
-ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'"
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'"
 rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/repo/target/release/"
 rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
+rsync -aL "$RPKI_CLIENT_LIBTLS_PATH" "$SSH_TARGET:$REMOTE_ROOT/lib/libtls.so.28"

 for idx in $(seq 1 "$ROUND_COUNT"); do
  ROUND_ID="$(printf 'round-%03d' "$idx")"
--- a/scripts/periodic/run_apnic_rpki_client_round_remote.sh
+++ b/scripts/periodic/run_apnic_rpki_client_round_remote.sh
@@ -87,8 +87,8 @@ mkdir -p "$REMOTE_OUT"
 if [[ "$KIND" == "snapshot" ]]; then
  rm -rf "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
 fi
-mkdir -p "$REMOTE_CACHE" "$REMOTE_STATE_OUT"
-chmod 0777 "$REMOTE_STATE_ROOT" "$REMOTE_CACHE" "$REMOTE_STATE_OUT"
+mkdir -p "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
+chmod -R 0777 "$REMOTE_STATE_ROOT"

 started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
 started_at_ms="$(python3 - <<'PY'
@@ -104,7 +104,8 @@ meta_out="$REMOTE_OUT/round-result.json"
 set +e
 (
  cd "$REMOTE_STATE_ROOT"
-  "$REMOTE_BIN" -vv -t "../../repo/tests/fixtures/tal/apnic-rfc7730-https.tal" -d "cache" "out"
+  LD_LIBRARY_PATH="$REMOTE_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \
+    "$REMOTE_BIN" -vv -t "../../repo/tests/fixtures/tal/apnic-rfc7730-https.tal" -d "cache" "out"
 ) >"$run_log" 2>&1
 exit_code=$?
 set -e
--- a/scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh
+++ b/scripts/periodic/run_arin_dual_rp_periodic_ccr_compare.sh
@@ -26,6 +26,7 @@ RUN_ROOT=""
 SSH_TARGET="${SSH_TARGET:-root@47.77.183.68}"
 REMOTE_ROOT=""
 RPKI_CLIENT_BIN="${RPKI_CLIENT_BIN:-/home/yuyr/dev/rpki-client-9.7/build-m5/src/rpki-client}"
+RPKI_CLIENT_LIBTLS_PATH="${RPKI_CLIENT_LIBTLS_PATH:-/home/yuyr/dev/rpki-client-9.7/.deps/libtls/root/usr/lib/x86_64-linux-gnu/libtls.so.28}"
 ROUND_COUNT=10
 INTERVAL_SECS=600
 START_AT=""
@@ -37,6 +38,7 @@ while [[ $# -gt 0 ]]; do
    --ssh-target) SSH_TARGET="$2"; shift 2 ;;
    --remote-root) REMOTE_ROOT="$2"; shift 2 ;;
    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --rpki-client-libtls) RPKI_CLIENT_LIBTLS_PATH="$2"; shift 2 ;;
    --round-count) ROUND_COUNT="$2"; shift 2 ;;
    --interval-secs) INTERVAL_SECS="$2"; shift 2 ;;
    --start-at) START_AT="$2"; shift 2 ;;
@@ -52,6 +54,7 @@ done
 if [[ "$DRY_RUN" -ne 1 ]]; then
  [[ -n "$REMOTE_ROOT" ]] || { echo "--remote-root is required unless --dry-run" >&2; exit 2; }
  [[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+  [[ -f "$RPKI_CLIENT_LIBTLS_PATH" ]] || { echo "rpki-client libtls not found: $RPKI_CLIENT_LIBTLS_PATH" >&2; exit 2; }
 fi

 mkdir -p "$RUN_ROOT"
@@ -180,9 +183,10 @@ rsync -a --delete \
  --exclude target \
  --exclude .git \
  "$ROOT_DIR/" "$SSH_TARGET:$REMOTE_ROOT/repo/"
-ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'"
+ssh "$SSH_TARGET" "mkdir -p '$REMOTE_ROOT/repo/target/release' '$REMOTE_ROOT/bin' '$REMOTE_ROOT/lib' '$REMOTE_ROOT/rounds' '$REMOTE_ROOT/state/ours' '$REMOTE_ROOT/state/rpki-client'"
 rsync -a "$ROOT_DIR/target/release/rpki" "$SSH_TARGET:$REMOTE_ROOT/repo/target/release/"
 rsync -a "$RPKI_CLIENT_BIN" "$SSH_TARGET:$REMOTE_ROOT/bin/rpki-client"
+rsync -aL "$RPKI_CLIENT_LIBTLS_PATH" "$SSH_TARGET:$REMOTE_ROOT/lib/libtls.so.28"

 for idx in $(seq 1 "$ROUND_COUNT"); do
  ROUND_ID="$(printf 'round-%03d' "$idx")"
--- a/scripts/periodic/run_arin_rpki_client_round_remote.sh
+++ b/scripts/periodic/run_arin_rpki_client_round_remote.sh
@@ -88,8 +88,8 @@ mkdir -p "$REMOTE_OUT"
 if [[ "$KIND" == "snapshot" ]]; then
  rm -rf "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
 fi
-mkdir -p "$REMOTE_CACHE" "$REMOTE_STATE_OUT"
-chmod 0777 "$REMOTE_STATE_ROOT" "$REMOTE_CACHE" "$REMOTE_STATE_OUT"
+mkdir -p "$REMOTE_CACHE" "$REMOTE_STATE_OUT" "$REMOTE_STATE_ROOT/ta" "$REMOTE_STATE_ROOT/.ta"
+chmod -R 0777 "$REMOTE_STATE_ROOT"

 started_at_iso="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
 started_at_ms="$(python3 - <<'PY'
@@ -106,7 +106,8 @@ meta_out="$REMOTE_OUT/round-result.json"
 set +e
 (
  cd "$REMOTE_STATE_ROOT"
-  "$REMOTE_BIN" -vv -t "../../repo/tests/fixtures/tal/arin.tal" -d "cache" "out"
+  LD_LIBRARY_PATH="$REMOTE_ROOT/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \
+    "$REMOTE_BIN" -vv -t "../../repo/tests/fixtures/tal/arin.tal" -d "cache" "out"
 ) >"$run_log" 2>&1
 exit_code=$?
 set -e