20260324 完成和routinator对齐snapshot delta replay correctness一致

2026-03-24 10:10:04 +08:00
parent 557a69cbd2
commit d6d44669b4
23 changed files with 2043 additions and 223 deletions
--- a/scripts/payload_replay/README.md
+++ b/scripts/payload_replay/README.md
@@ -53,7 +53,7 @@ python3 scripts/payload_replay/multi_rir_case_info.py \

 - 从 multi-RIR bundle 中选择指定 RIR 的 snapshot/base 与 delta 输入
 - 读取该 RIR 的 Routinator `base-replay` / `delta-replay` timing 基线
- 使用该 RIR `timings/base-replay.json` 与 `timings/delta-replay.json` 的 `startedAt` 作为 replay `--validation-time`
+- 优先使用 `base-locks.json.validationTime` 与 `locks-delta.json.validationTime` 作为 replay `--validation-time`；若缺失才回退到 `timings/base-replay.json` 与 `timings/delta-replay.json` 的 `startedAt`
 - 在 `target/live/multi_rir_replay_runs/<rir>/` 下生成：
  - snapshot replay 产物
  - delta replay 产物
@@ -65,6 +65,40 @@ python3 scripts/payload_replay/multi_rir_case_info.py \

 也可以通过 `BUNDLE_ROOT` 覆盖。

+## `run_apnic_snapshot_replay_profile.sh`
+
+基于 multi-RIR bundle 中的 APNIC snapshot 输入，使用当前 replay 主流程执行一次带 `--analyze` 和 `--profile-cpu` 的离线 profile。
+
+```bash
+./scripts/payload_replay/run_apnic_snapshot_replay_profile.sh
+```
+
+作用：
+
+- 使用 `APNIC` 的 snapshot/base replay 输入
+- 自动开启：
+  - `--analyze`
+  - `--profile-cpu`
+- 自动记录：
+  - replay wall-clock 时长
+  - Routinator baseline (`base-replay`)
+  - analyze 目录路径
+- 生成：
+  - `report.json`
+  - `run.log`
+  - `meta.json`
+  - `summary.md`
+  - 以及 `target/live/analyze/<timestamp>/` 下的：
+    - `timing.json`
+    - `flamegraph.svg`
+    - `pprof.pb.gz`
+
+支持：
+
+- `DRY_RUN=1`：只打印命令，不真正执行
+- `MAX_DEPTH` / `MAX_INSTANCES`：用于限定 replay 范围
+- `PROFILE_RUN_ROOT`：覆盖 wrapper 产物输出目录
+
 ## `run_apnic_replay.sh`

 默认使用：
--- a/scripts/payload_replay/multi_rir_case_info.py
+++ b/scripts/payload_replay/multi_rir_case_info.py
@@ -58,6 +58,14 @@ def load_timing_summary(bundle_root: Path) -> dict:
    return json.loads(timing_path.read_text(encoding="utf-8"))


+def load_json(path: Path) -> dict:
+    return json.loads(require_path(path, "file").read_text(encoding="utf-8"))
+
+
+def lock_validation_time(lock_obj: dict, fallback_started_at: str) -> str:
+    return lock_obj.get("validationTime") or lock_obj.get("validation_time") or fallback_started_at
+
+
 def build_case(bundle_root: Path, repo_root: Path, rir: str) -> dict:
    if rir not in RIR_CONFIG:
        raise SystemExit(
@@ -76,6 +84,8 @@ def build_case(bundle_root: Path, repo_root: Path, rir: str) -> dict:
    delta_timing = require_path(rir_root / "timings" / "delta-replay.json", "file")
    base_timing_obj = json.loads(base_timing.read_text(encoding="utf-8"))
    delta_timing_obj = json.loads(delta_timing.read_text(encoding="utf-8"))
+    base_locks_obj = load_json(rir_root / "base-locks.json")
+    delta_locks_obj = load_json(rir_root / "locks-delta.json")

    case = {
        "bundle_root": str(bundle_root),
@@ -98,8 +108,12 @@ def build_case(bundle_root: Path, repo_root: Path, rir: str) -> dict:
        "tal_path": str(require_path(repo_root / cfg["tal"], "file")),
        "ta_path": str(require_path(repo_root / cfg["ta"], "file")),
        "validation_times": {
-            "snapshot": base_timing_obj["startedAt"],
-            "delta": delta_timing_obj["startedAt"],
+            "snapshot": lock_validation_time(base_locks_obj, base_timing_obj["startedAt"]),
+            "delta": lock_validation_time(delta_locks_obj, delta_timing_obj["startedAt"]),
+        },
+        "timing_started_at": {
+            "snapshot_replay": base_timing_obj["startedAt"],
+            "delta_replay": delta_timing_obj["startedAt"],
        },
        "routinator_timings": {
            "base_replay_seconds": float(durations["base-replay"]),
--- a/scripts/payload_replay/run_apnic_delta_replay.sh
+++ b/scripts/payload_replay/run_apnic_delta_replay.sh
@@ -11,7 +11,7 @@ PAYLOAD_BASE_ARCHIVE="${PAYLOAD_BASE_ARCHIVE:-$DELTA_ROOT/base-payload-archive}"
 PAYLOAD_BASE_LOCKS="${PAYLOAD_BASE_LOCKS:-$DELTA_ROOT/base-locks.json}"
 PAYLOAD_DELTA_ARCHIVE="${PAYLOAD_DELTA_ARCHIVE:-$DELTA_ROOT/payload-delta-archive}"
 PAYLOAD_DELTA_LOCKS="${PAYLOAD_DELTA_LOCKS:-$DELTA_ROOT/locks-delta.json}"
-VALIDATION_TIME="${VALIDATION_TIME:-2026-03-15T10:00:00Z}"
+VALIDATION_TIME="${VALIDATION_TIME:-}"
 PAYLOAD_BASE_VALIDATION_TIME="${PAYLOAD_BASE_VALIDATION_TIME:-}"
 TRUST_ANCHOR="${TRUST_ANCHOR:-apnic}"
 ROUTINATOR_RECORD_CSV="${ROUTINATOR_RECORD_CSV:-$DELTA_ROOT/record-delta.csv}"
@@ -20,6 +20,28 @@ MAX_INSTANCES="${MAX_INSTANCES:-}"
 OUT_DIR="${OUT_DIR:-$ROOT_DIR/target/live/payload_delta_replay_runs}"
 mkdir -p "$OUT_DIR"

+if [[ -z "$PAYLOAD_BASE_VALIDATION_TIME" ]]; then
+  PAYLOAD_BASE_VALIDATION_TIME="$(python3 - "$PAYLOAD_BASE_LOCKS" <<'LOCKPY'
+import json, sys
+from pathlib import Path
+path = Path(sys.argv[1])
+data = json.loads(path.read_text(encoding='utf-8'))
+print(data.get('validationTime') or data.get('validation_time') or '')
+LOCKPY
+)"
+fi
+
+if [[ -z "$VALIDATION_TIME" ]]; then
+  VALIDATION_TIME="$(python3 - "$PAYLOAD_DELTA_LOCKS" <<'LOCKPY'
+import json, sys
+from pathlib import Path
+path = Path(sys.argv[1])
+data = json.loads(path.read_text(encoding='utf-8'))
+print(data.get('validationTime') or data.get('validation_time') or '2026-03-15T10:00:00Z')
+LOCKPY
+)"
+fi
+
 TS="$(date -u +%Y%m%dT%H%M%SZ)"
 RUN_NAME="${RUN_NAME:-apnic_delta_replay_${TS}}"
 DB_DIR="${DB_DIR:-$OUT_DIR/${RUN_NAME}_db}"
--- a/scripts/payload_replay/run_apnic_replay.sh
+++ b/scripts/payload_replay/run_apnic_replay.sh
@@ -8,7 +8,7 @@ TAL_PATH="${TAL_PATH:-$ROOT_DIR/tests/fixtures/tal/apnic-rfc7730-https.tal}"
 TA_PATH="${TA_PATH:-$ROOT_DIR/tests/fixtures/ta/apnic-ta.cer}"
 PAYLOAD_REPLAY_ARCHIVE="${PAYLOAD_REPLAY_ARCHIVE:-$ROOT_DIR/target/live/payload_replay/payload-archive}"
 PAYLOAD_REPLAY_LOCKS="${PAYLOAD_REPLAY_LOCKS:-$ROOT_DIR/target/live/payload_replay/locks.json}"
-VALIDATION_TIME="${VALIDATION_TIME:-2026-03-13T02:30:00Z}"
+VALIDATION_TIME="${VALIDATION_TIME:-}"
 TRUST_ANCHOR="${TRUST_ANCHOR:-apnic}"
 ROUTINATOR_RECORD_CSV="${ROUTINATOR_RECORD_CSV:-$ROOT_DIR/target/live/payload_replay/record.csv}"
 MAX_DEPTH="${MAX_DEPTH:-}"
@@ -16,6 +16,17 @@ MAX_INSTANCES="${MAX_INSTANCES:-}"
 OUT_DIR="${OUT_DIR:-$ROOT_DIR/target/live/payload_replay_runs}"
 mkdir -p "$OUT_DIR"

+if [[ -z "$VALIDATION_TIME" ]]; then
+  VALIDATION_TIME="$(python3 - "$PAYLOAD_REPLAY_LOCKS" <<'LOCKPY'
+import json, sys
+from pathlib import Path
+path = Path(sys.argv[1])
+data = json.loads(path.read_text(encoding='utf-8'))
+print(data.get('validationTime') or data.get('validation_time') or '2026-03-13T02:30:00Z')
+LOCKPY
+)"
+fi
+
 TS="$(date -u +%Y%m%dT%H%M%SZ)"
 RUN_NAME="${RUN_NAME:-apnic_replay_${TS}}"
 DB_DIR="${DB_DIR:-$OUT_DIR/${RUN_NAME}_db}"
--- a/scripts/payload_replay/run_apnic_snapshot_replay_profile.sh
+++ b/scripts/payload_replay/run_apnic_snapshot_replay_profile.sh
@@ -0,0 +1,168 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "$ROOT_DIR"
+
+BUNDLE_ROOT="${BUNDLE_ROOT:-$ROOT_DIR/../../rpki/target/live/20260316-112341-multi-final3}"
+CASE_INFO_SCRIPT="$ROOT_DIR/scripts/payload_replay/multi_rir_case_info.py"
+PROFILE_RUN_ROOT="${PROFILE_RUN_ROOT:-$ROOT_DIR/target/live/analyze_runs}"
+mkdir -p "$PROFILE_RUN_ROOT"
+
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+RUN_NAME="${RUN_NAME:-apnic_snapshot_profile_${TS}}"
+RUN_DIR="$PROFILE_RUN_ROOT/$RUN_NAME"
+mkdir -p "$RUN_DIR"
+
+ANALYZE_ROOT="$ROOT_DIR/target/live/analyze"
+mkdir -p "$ANALYZE_ROOT"
+mapfile -t ANALYZE_BEFORE < <(find "$ANALYZE_ROOT" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort)
+
+eval "$(python3 "$CASE_INFO_SCRIPT" --bundle-root "$BUNDLE_ROOT" --rir apnic --format env)"
+
+DB_DIR="${DB_DIR:-$RUN_DIR/db}"
+REPORT_JSON="${REPORT_JSON:-$RUN_DIR/report.json}"
+RUN_LOG="${RUN_LOG:-$RUN_DIR/run.log}"
+META_JSON="${META_JSON:-$RUN_DIR/meta.json}"
+SUMMARY_MD="${SUMMARY_MD:-$RUN_DIR/summary.md}"
+
+rm -rf "$DB_DIR"
+
+cmd=(cargo run --release --features profile --bin rpki --
+  --db "$DB_DIR"
+  --tal-path "$TAL_PATH"
+  --ta-path "$TA_PATH"
+  --payload-replay-archive "$PAYLOAD_REPLAY_ARCHIVE"
+  --payload-replay-locks "$PAYLOAD_REPLAY_LOCKS"
+  --validation-time "$SNAPSHOT_VALIDATION_TIME"
+  --analyze
+  --profile-cpu
+  --report-json "$REPORT_JSON")
+
+if [[ -n "${MAX_DEPTH:-}" ]]; then
+  cmd+=(--max-depth "$MAX_DEPTH")
+fi
+if [[ -n "${MAX_INSTANCES:-}" ]]; then
+  cmd+=(--max-instances "$MAX_INSTANCES")
+fi
+
+if [[ "${DRY_RUN:-0}" == "1" ]]; then
+  printf '%q ' "${cmd[@]}"
+  echo
+  exit 0
+fi
+
+run_start_s="$(date +%s)"
+(
+  echo '# command:'
+  printf '%q ' "${cmd[@]}"
+  echo
+  echo
+  "${cmd[@]}"
+) 2>&1 | tee "$RUN_LOG" >/dev/null
+run_end_s="$(date +%s)"
+run_duration_s="$((run_end_s - run_start_s))"
+
+mapfile -t ANALYZE_AFTER < <(find "$ANALYZE_ROOT" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort)
+ANALYZE_DIR=""
+for candidate in "${ANALYZE_AFTER[@]}"; do
+  seen=0
+  for old in "${ANALYZE_BEFORE[@]}"; do
+    if [[ "$candidate" == "$old" ]]; then
+      seen=1
+      break
+    fi
+  done
+  if [[ "$seen" == "0" ]]; then
+    ANALYZE_DIR="$candidate"
+  fi
+done
+if [[ -z "$ANALYZE_DIR" ]]; then
+  ANALYZE_DIR="$(find "$ANALYZE_ROOT" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort | tail -n 1)"
+fi
+
+BUNDLE_ROOT="$BUNDLE_ROOT" \
+TRUST_ANCHOR="$TRUST_ANCHOR" \
+TAL_PATH="$TAL_PATH" \
+TA_PATH="$TA_PATH" \
+PAYLOAD_REPLAY_ARCHIVE="$PAYLOAD_REPLAY_ARCHIVE" \
+PAYLOAD_REPLAY_LOCKS="$PAYLOAD_REPLAY_LOCKS" \
+SNAPSHOT_VALIDATION_TIME="$SNAPSHOT_VALIDATION_TIME" \
+ROUTINATOR_BASE_REPLAY_SECONDS="$ROUTINATOR_BASE_REPLAY_SECONDS" \
+DB_DIR="$DB_DIR" \
+REPORT_JSON="$REPORT_JSON" \
+RUN_LOG="$RUN_LOG" \
+ANALYZE_DIR="$ANALYZE_DIR" \
+RUN_DURATION_S="$run_duration_s" \
+python3 - "$META_JSON" "$SUMMARY_MD" <<'PY'
+import json
+import os
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+meta_path = Path(sys.argv[1])
+summary_path = Path(sys.argv[2])
+report_path = Path(os.environ['REPORT_JSON'])
+report = json.loads(report_path.read_text(encoding='utf-8'))
+recorded = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')
+meta = {
+    'recorded_at_utc': recorded,
+    'bundle_root': os.environ['BUNDLE_ROOT'],
+    'trust_anchor': os.environ['TRUST_ANCHOR'],
+    'tal_path': os.environ['TAL_PATH'],
+    'ta_path': os.environ['TA_PATH'],
+    'payload_replay_archive': os.environ['PAYLOAD_REPLAY_ARCHIVE'],
+    'payload_replay_locks': os.environ['PAYLOAD_REPLAY_LOCKS'],
+    'validation_time_arg': os.environ['SNAPSHOT_VALIDATION_TIME'],
+    'routinator_base_replay_seconds': float(os.environ['ROUTINATOR_BASE_REPLAY_SECONDS']),
+    'db_dir': os.environ['DB_DIR'],
+    'report_json': os.environ['REPORT_JSON'],
+    'run_log': os.environ['RUN_LOG'],
+    'analyze_dir': os.environ.get('ANALYZE_DIR') or '',
+    'durations_secs': {
+        'rpki_run_wall': int(os.environ['RUN_DURATION_S']),
+    },
+    'counts': {
+        'publication_points_processed': report['tree']['instances_processed'],
+        'publication_points_failed': report['tree']['instances_failed'],
+        'vrps': len(report['vrps']),
+        'aspas': len(report['aspas']),
+        'audit_publication_points': len(report['publication_points']),
+    },
+}
+meta_path.write_text(json.dumps(meta, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
+ratio = meta['durations_secs']['rpki_run_wall'] / meta['routinator_base_replay_seconds'] if meta['routinator_base_replay_seconds'] else None
+lines = []
+lines.append('# APNIC Snapshot Replay Profile Summary\n\n')
+lines.append(f"- recorded_at_utc: `{recorded}`\n")
+lines.append(f"- bundle_root: `{meta['bundle_root']}`\n")
+lines.append(f"- tal_path: `{meta['tal_path']}`\n")
+lines.append(f"- ta_path: `{meta['ta_path']}`\n")
+lines.append(f"- payload_replay_archive: `{meta['payload_replay_archive']}`\n")
+lines.append(f"- payload_replay_locks: `{meta['payload_replay_locks']}`\n")
+lines.append(f"- validation_time_arg: `{meta['validation_time_arg']}`\n")
+lines.append(f"- db_dir: `{meta['db_dir']}`\n")
+lines.append(f"- report_json: `{meta['report_json']}`\n")
+lines.append(f"- run_log: `{meta['run_log']}`\n")
+lines.append(f"- analyze_dir: `{meta['analyze_dir']}`\n\n")
+lines.append('## Timing\n\n')
+lines.append('| metric | value |\n')
+lines.append('|---|---:|\n')
+lines.append(f"| ours_snapshot_replay_wall_s | {meta['durations_secs']['rpki_run_wall']} |\n")
+lines.append(f"| routinator_base_replay_s | {meta['routinator_base_replay_seconds']:.3f} |\n")
+if ratio is not None:
+    lines.append(f"| ratio_ours_over_routinator | {ratio:.3f} |\n")
+lines.append('\n## Counts\n\n')
+for key, value in meta['counts'].items():
+    lines.append(f"- {key}: `{value}`\n")
+summary_path.write_text(''.join(lines), encoding='utf-8')
+PY
+
+echo "== APNIC snapshot replay profiling complete ==" >&2
+echo "- run_dir: $RUN_DIR" >&2
+echo "- analyze_dir: $ANALYZE_DIR" >&2
+echo "- report_json: $REPORT_JSON" >&2
+echo "- run_log: $RUN_LOG" >&2
+echo "- meta_json: $META_JSON" >&2
+echo "- summary_md: $SUMMARY_MD" >&2