20260716 增加verification-only缓存正确性复核

2026-07-16 19:08:16 +08:00
parent 7aac4855bd
commit 59623b449b
14 changed files with 2269 additions and 19 deletions
--- a/scripts/soak/build_portable_soak_package.sh
+++ b/scripts/soak/build_portable_soak_package.sh
@@ -14,7 +14,7 @@ Usage:
  scripts/soak/build_portable_soak_package.sh [--out-dir <path>] [--profile <profile>]

 Requires release binaries to already exist. Build them first, for example:
-  cargo build --release --bin rpki --bin rpki_daemon --bin db_stats --bin rpki_artifact_metrics --bin rpki_query_service --bin rpki_query_indexer
+  cargo build --release --bin rpki --bin rpki_daemon --bin db_stats --bin rpki_artifact_metrics --bin rpki_query_service --bin rpki_query_indexer --bin verification_ccr_compare
 USAGE
 }

@@ -53,7 +53,7 @@ else
  TARGET_BIN_DIR="$REPO_ROOT/target/$PROFILE"
 fi

-REQUIRED_BINS=(rpki rpki_daemon db_stats rpki_artifact_metrics rpki_query_service rpki_query_indexer)
+REQUIRED_BINS=(rpki rpki_daemon db_stats rpki_artifact_metrics rpki_query_service rpki_query_indexer verification_ccr_compare)
 OPTIONAL_BINS=(
  ccr_dump
  ccr_state_compare
@@ -82,7 +82,7 @@ STAGE_DIR="$BUILD_ROOT/$PACKAGE_DIR_NAME"
 ARCHIVE_PATH="$OUT_DIR/$PACKAGE_NAME.tar.gz"

 rm -rf "$STAGE_DIR" "$ARCHIVE_PATH"
-mkdir -p "$STAGE_DIR/bin" "$STAGE_DIR/fixtures" "$STAGE_DIR/scripts" "$STAGE_DIR/scripts/soak" \
+mkdir -p "$STAGE_DIR/bin" "$STAGE_DIR/fixtures" "$STAGE_DIR/scripts" "$STAGE_DIR/scripts/soak" "$STAGE_DIR/scripts/verification" \
  "$STAGE_DIR/runs" "$STAGE_DIR/state" "$STAGE_DIR/logs" "$STAGE_DIR/tmp"

 install -m 0755 "$SCRIPT_DIR/run_soak.sh" "$STAGE_DIR/run_soak.sh"
@@ -92,6 +92,8 @@ install -m 0755 "$SCRIPT_DIR/fixed_phase_loop.sh" "$STAGE_DIR/scripts/soak/fixed
 install -m 0755 "$SCRIPT_DIR/hourly_soak_report.py" "$STAGE_DIR/scripts/soak/hourly_soak_report.py"
 install -m 0755 "$SCRIPT_DIR/publish_remote231.sh" "$STAGE_DIR/scripts/soak/publish_remote231.sh"
 install -m 0755 "$SCRIPT_DIR/publish_remote231_full.sh" "$STAGE_DIR/scripts/soak/publish_remote231_full.sh"
+install -m 0755 "$REPO_ROOT/scripts/verification/run_verification_only.sh" "$STAGE_DIR/scripts/verification/run_verification_only.sh"
+install -m 0644 "$REPO_ROOT/scripts/verification/README.md" "$STAGE_DIR/scripts/verification/README.md"
 install -m 0644 "$SCRIPT_DIR/portable-soak.env.example" "$STAGE_DIR/.env"
 install -m 0644 "$SCRIPT_DIR/portable-soak.env.example" "$STAGE_DIR/portable-soak.env.example"

--- a/scripts/soak/run_soak.sh
+++ b/scripts/soak/run_soak.sh
@@ -48,6 +48,7 @@ TMP_DIR="${TMP_DIR:-$RUN_ROOT/tmp}"
 RSYNC_MIRROR_ROOT="${RSYNC_MIRROR_ROOT:-$STATE_ROOT/rsync-mirror}"
 INVALID_ROOT="$STATE_ROOT/invalid"
 RESET_STAGING_ROOT="$STATE_ROOT/reset-staging"
+STATE_EXECUTION_LOCK_PATH="$STATE_ROOT/locks/state-execution.lock"
 LIVE_TA_REFRESH_DIR="${LIVE_TA_REFRESH_DIR:-$META_DIR/live-ta-refresh}"
 LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS="${LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS:-15}"
 LIVE_TA_REFRESH_MAX_TIME_SECS="${LIVE_TA_REFRESH_MAX_TIME_SECS:-120}"
@@ -1080,6 +1081,7 @@ build_child_args() {

  CHILD_ARGS+=(
    --report-json "{run_out}/report.json"
+    --validation-contract-out "{run_out}/validation-contract.json"
  )
  if is_true "$OUTPUT_COMPACT_REPORT"; then
    CHILD_ARGS+=(--report-json-compact)
@@ -1109,6 +1111,59 @@ build_child_args() {
  fi
 }

+write_current_state_binding() {
+  local run_dir="$1"
+  local run_id="$2"
+  local output_path="$META_DIR/current-state-binding.json"
+  python3 - "$run_dir" "$run_id" "$output_path" <<'PY'
+import hashlib
+import json
+import os
+import sys
+import tempfile
+from datetime import datetime, timezone
+from pathlib import Path
+
+run_dir = Path(sys.argv[1])
+run_id = sys.argv[2]
+output_path = Path(sys.argv[3])
+summary = json.loads((run_dir / "run-summary.json").read_text(encoding="utf-8"))
+if summary.get("status") != "success":
+    raise SystemExit(f"cannot bind unsuccessful source run: {run_id}")
+
+def digest(name):
+    path = run_dir / name
+    if not path.is_file():
+        raise SystemExit(f"current state binding input is missing: {path}")
+    hasher = hashlib.sha256()
+    with path.open("rb") as handle:
+        for chunk in iter(lambda: handle.read(1024 * 1024), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+binding = {
+    "schemaVersion": 1,
+    "currentRunId": run_id,
+    "sourceCirSha256": digest("input.cir"),
+    "sourceCcrSha256": digest("result.ccr"),
+    "validationContractSha256": digest("validation-contract.json"),
+    "updatedAtRfc3339Utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
+}
+output_path.parent.mkdir(parents=True, exist_ok=True)
+fd, tmp_name = tempfile.mkstemp(prefix=output_path.name + ".", dir=output_path.parent)
+try:
+    with os.fdopen(fd, "w", encoding="utf-8") as handle:
+        json.dump(binding, handle, indent=2, sort_keys=True)
+        handle.write("\n")
+        handle.flush()
+        os.fsync(handle.fileno())
+    os.replace(tmp_name, output_path)
+finally:
+    if os.path.exists(tmp_name):
+        os.unlink(tmp_name)
+PY
+}
+
 copy_inner_run_outputs() {
  local daemon_state_root="$1"
  local run_dir="$2"
@@ -1304,6 +1359,12 @@ run_one_round() {
      final_status="failed"
    fi
  fi
+  if [[ "$final_status" == "success" ]]; then
+    if ! write_current_state_binding "$run_dir" "$run_id"; then
+      warn "failed to bind successful run to current state: $run_id"
+      final_status="failed"
+    fi
+  fi
  write_run_meta "$run_dir/run-meta.json" "$final_status" "$run_index" "$run_id" "$sync_mode" \
    "$snapshot_reason" "$previous_run_id" "$previous_success_value" "$started_at" "$completed_at" \
    "$INVALID_DB_PATH" "$INVALID_STATE_PATH" "$INVALID_TMP_PATH" "$daemon_exit_code" "$PACKAGE_ROOT" "$ENV_FILE" \
@@ -1327,6 +1388,7 @@ main() {
  require_command python3
  require_command date
  require_command find
+  require_command flock
  if [[ "$TAL_INPUT_MODE" == "file-live-ta" ]]; then
    require_command curl
    validate_positive_int "LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS" "$LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS"
@@ -1360,7 +1422,7 @@ main() {
    fi
  done

-  mkdir -p "$RUNS_ROOT" "$LOG_ROOT" "$DB_DIR" "$META_DIR" "$TMP_DIR" "$INVALID_ROOT" "$RESET_STAGING_ROOT" "$LIVE_TA_REFRESH_DIR"
+  mkdir -p "$RUNS_ROOT" "$LOG_ROOT" "$DB_DIR" "$META_DIR" "$TMP_DIR" "$INVALID_ROOT" "$RESET_STAGING_ROOT" "$LIVE_TA_REFRESH_DIR" "$(dirname "$STATE_EXECUTION_LOCK_PATH")"
  if is_true "$ALLOW_RSYNC_MIRROR_REUSE"; then
    mkdir -p "$RSYNC_MIRROR_ROOT"
  fi
@@ -1386,6 +1448,9 @@ main() {
  local any_failed=0

  while (( run_forever == 1 || next_index <= stop_index )); do
+    local state_lock_fd
+    exec {state_lock_fd}>"$STATE_EXECUTION_LOCK_PATH"
+    flock "$state_lock_fd"
    INVALID_DB_PATH=""
    INVALID_STATE_PATH=""
    INVALID_TMP_PATH=""
@@ -1472,6 +1537,8 @@ main() {
      echo "completed run $(printf 'run_%04d' "$next_index") status=failed" >&2
      any_failed=1
    fi
+    flock -u "$state_lock_fd"
+    exec {state_lock_fd}>&-
    if (( STOP_AFTER_SECS > 0 )); then
      elapsed_secs=$(( $(date +%s) - started_epoch ))
      if (( elapsed_secs >= STOP_AFTER_SECS )); then
--- a/scripts/verification/README.md
+++ b/scripts/verification/README.md
@@ -0,0 +1,50 @@
+# verification-only 使用说明
+
+`verification-only` 用于复核启用验证缓存的最新成功 normal run。它冻结当前仓库视图，关闭 PP、ROA、child certificate cache、prefetch、VCIR 持久化和网络访问，在源 CIR 的同一验证时间重新执行完整验证，并比较 CCR 的 MFT、VRP、VAP、TA、router key 五类 state digest。
+
+## 前提
+
+- 源 run 必须是当前 state DB 对应的最新成功 normal run。
+- 源 run 必须包含 `input.cir`、`result.ccr`、`run-summary.json` 和 `validation-contract.json`。
+- state root 必须包含 `db/work-db`、`db/repo-bytes.db` 和 `meta/current-state-binding.json`。
+- 源 normal run 必须启用至少一种验证缓存；仅启用 transport prefetch 不算验证缓存。
+- verification-only 与 normal soak 共用 `state/locks/state-execution.lock`，不能绕过 wrapper 并发执行。
+
+## 执行
+
+在 portable package 根目录执行：
+
+```bash
+./scripts/verification/run_verification_only.sh \
+  --source-run-dir /data/ours-rp/runs/run_0123 \
+  --source-state-root /data/ours-rp/state \
+  --out-dir /data/ours-rp/verification-runs/run_0123
+```
+
+验证时间自动从源 `input.cir` 读取，不接受命令行覆盖。成功后 scratch work-db 默认删除；排障时可加 `--keep-scratch`。
+
+## 输出
+
+- `input.cir`、`result.ccr`、`report.json`
+- `vrps.csv`、`vaps.csv`
+- `stage-timing.json`、`validation-events.jsonl`
+- `validation-contract.json`、`verification-meta.json`
+- `compare/ccr-state-compare.json`、`compare/ccr-state-compare.md`
+- `stdout.log`、`stderr.log`、`process-time.txt`、`wrapper-timing.json`
+
+通过条件是 compare JSON 中 `stateDigestMatch=true`。`producedAt` 不参与比较。
+
+## 隔离保证
+
+- 不分配 normal run id，不更新 lifecycle、periodic snapshot 计数和 retention。
+- 源 work-db 只用于创建 checkpoint，后续写入只发生在独立 scratch work-db。
+- 源 repo-bytes.db 以只读模式打开。
+- 输出目录必须不存在，防止覆盖旧验证证据。
+
+## 常见失败
+
+- `current state binding`：所选 run 已不是当前 state 对应的最新成功 run，需选择 binding 指向的 run。
+- `differs from current state binding`：源 CIR、CCR 或 contract 已变化，不能继续复核。
+- `blob is missing`：repository view 引用的原始 bytes 不完整，验证不会启动。
+- `did not enable a validation cache`：源 run 不是缓存正确性对比的有效基准。
+- `CCR state digest mismatch`：查看 `compare/` 下 JSON/Markdown，按不匹配状态集合继续排查。
--- a/scripts/verification/run_verification_only.sh
+++ b/scripts/verification/run_verification_only.sh
@@ -0,0 +1,102 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'USAGE'
+Usage:
+  run_verification_only.sh \
+    --source-run-dir <run_dir> \
+    --source-state-root <state_dir> \
+    --out-dir <verification_dir> \
+    [--rpki-bin <path>] [--keep-scratch]
+
+validation time is always read from the source CIR. The wrapper serializes access
+with normal soak runs and never writes into the normal run sequence.
+USAGE
+}
+
+SOURCE_RUN_DIR=""
+SOURCE_STATE_ROOT=""
+OUT_DIR=""
+RPKI_BIN=""
+KEEP_SCRATCH=0
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --source-run-dir) SOURCE_RUN_DIR="${2:?missing value}"; shift 2 ;;
+    --source-state-root) SOURCE_STATE_ROOT="${2:?missing value}"; shift 2 ;;
+    --out-dir) OUT_DIR="${2:?missing value}"; shift 2 ;;
+    --rpki-bin) RPKI_BIN="${2:?missing value}"; shift 2 ;;
+    --keep-scratch) KEEP_SCRATCH=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$SOURCE_RUN_DIR" ]] || { echo "--source-run-dir is required" >&2; exit 2; }
+[[ -n "$SOURCE_STATE_ROOT" ]] || { echo "--source-state-root is required" >&2; exit 2; }
+[[ -n "$OUT_DIR" ]] || { echo "--out-dir is required" >&2; exit 2; }
+if [[ -z "$RPKI_BIN" ]]; then
+  RPKI_BIN="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)/target/release/rpki"
+fi
+[[ -x "$RPKI_BIN" ]] || { echo "rpki binary is not executable: $RPKI_BIN" >&2; exit 2; }
+command -v flock >/dev/null || { echo "flock is required" >&2; exit 2; }
+command -v python3 >/dev/null || { echo "python3 is required" >&2; exit 2; }
+
+now_ms() {
+  python3 -c 'import time; print(time.time_ns() // 1_000_000)'
+}
+
+mkdir -p "$SOURCE_STATE_ROOT/locks" "$(dirname "$OUT_DIR")"
+[[ ! -e "$OUT_DIR" ]] || { echo "verification output already exists: $OUT_DIR" >&2; exit 2; }
+LOCK_PATH="$SOURCE_STATE_ROOT/locks/state-execution.lock"
+STDOUT_TMP="${OUT_DIR}.stdout.tmp"
+STDERR_TMP="${OUT_DIR}.stderr.tmp"
+PROCESS_TIME_TMP="${OUT_DIR}.process-time.tmp"
+WAIT_STARTED_MS="$(now_ms)"
+exec {LOCK_FD}>"$LOCK_PATH"
+flock "$LOCK_FD"
+LOCK_ACQUIRED_MS="$(now_ms)"
+
+ARGS=(
+  --verification-only
+  --verification-source-run "$SOURCE_RUN_DIR"
+  --verification-source-state "$SOURCE_STATE_ROOT"
+  --verification-out "$OUT_DIR"
+)
+if [[ "$KEEP_SCRATCH" -eq 1 ]]; then
+  ARGS+=(--keep-verification-scratch)
+fi
+
+set +e
+if [[ -x /usr/bin/time ]]; then
+  RPKI_STATE_EXECUTION_LOCK_HELD=1 /usr/bin/time -v -o "$PROCESS_TIME_TMP" -- \
+    "$RPKI_BIN" "${ARGS[@]}" >"$STDOUT_TMP" 2>"$STDERR_TMP"
+else
+  RPKI_STATE_EXECUTION_LOCK_HELD=1 "$RPKI_BIN" "${ARGS[@]}" >"$STDOUT_TMP" 2>"$STDERR_TMP"
+fi
+EXIT_CODE=$?
+set -e
+LOCK_RELEASED_MS="$(now_ms)"
+flock -u "$LOCK_FD"
+exec {LOCK_FD}>&-
+
+mkdir -p "$OUT_DIR"
+mv "$STDOUT_TMP" "$OUT_DIR/stdout.log"
+mv "$STDERR_TMP" "$OUT_DIR/stderr.log"
+if [[ -f "$PROCESS_TIME_TMP" ]]; then
+  mv "$PROCESS_TIME_TMP" "$OUT_DIR/process-time.txt"
+fi
+python3 - "$OUT_DIR/wrapper-timing.json" "$WAIT_STARTED_MS" "$LOCK_ACQUIRED_MS" "$LOCK_RELEASED_MS" "$EXIT_CODE" <<'PY'
+import json
+import sys
+from pathlib import Path
+
+path = Path(sys.argv[1])
+wait_started_ms, lock_acquired_ms, lock_released_ms, exit_code = map(int, sys.argv[2:])
+path.write_text(json.dumps({
+    "lockWaitMs": lock_acquired_ms - wait_started_ms,
+    "lockHeldMs": lock_released_ms - lock_acquired_ms,
+    "exitCode": exit_code,
+}, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+PY
+exit "$EXIT_CODE"