20260716 增加verification-only缓存正确性复核
This commit is contained in:
@@ -14,7 +14,7 @@ Usage:
|
||||
scripts/soak/build_portable_soak_package.sh [--out-dir <path>] [--profile <profile>]
|
||||
|
||||
Requires release binaries to already exist. Build them first, for example:
|
||||
cargo build --release --bin rpki --bin rpki_daemon --bin db_stats --bin rpki_artifact_metrics --bin rpki_query_service --bin rpki_query_indexer
|
||||
cargo build --release --bin rpki --bin rpki_daemon --bin db_stats --bin rpki_artifact_metrics --bin rpki_query_service --bin rpki_query_indexer --bin verification_ccr_compare
|
||||
USAGE
|
||||
}
|
||||
|
||||
@@ -53,7 +53,7 @@ else
|
||||
TARGET_BIN_DIR="$REPO_ROOT/target/$PROFILE"
|
||||
fi
|
||||
|
||||
REQUIRED_BINS=(rpki rpki_daemon db_stats rpki_artifact_metrics rpki_query_service rpki_query_indexer)
|
||||
REQUIRED_BINS=(rpki rpki_daemon db_stats rpki_artifact_metrics rpki_query_service rpki_query_indexer verification_ccr_compare)
|
||||
OPTIONAL_BINS=(
|
||||
ccr_dump
|
||||
ccr_state_compare
|
||||
@@ -82,7 +82,7 @@ STAGE_DIR="$BUILD_ROOT/$PACKAGE_DIR_NAME"
|
||||
ARCHIVE_PATH="$OUT_DIR/$PACKAGE_NAME.tar.gz"
|
||||
|
||||
rm -rf "$STAGE_DIR" "$ARCHIVE_PATH"
|
||||
mkdir -p "$STAGE_DIR/bin" "$STAGE_DIR/fixtures" "$STAGE_DIR/scripts" "$STAGE_DIR/scripts/soak" \
|
||||
mkdir -p "$STAGE_DIR/bin" "$STAGE_DIR/fixtures" "$STAGE_DIR/scripts" "$STAGE_DIR/scripts/soak" "$STAGE_DIR/scripts/verification" \
|
||||
"$STAGE_DIR/runs" "$STAGE_DIR/state" "$STAGE_DIR/logs" "$STAGE_DIR/tmp"
|
||||
|
||||
install -m 0755 "$SCRIPT_DIR/run_soak.sh" "$STAGE_DIR/run_soak.sh"
|
||||
@@ -92,6 +92,8 @@ install -m 0755 "$SCRIPT_DIR/fixed_phase_loop.sh" "$STAGE_DIR/scripts/soak/fixed
|
||||
install -m 0755 "$SCRIPT_DIR/hourly_soak_report.py" "$STAGE_DIR/scripts/soak/hourly_soak_report.py"
|
||||
install -m 0755 "$SCRIPT_DIR/publish_remote231.sh" "$STAGE_DIR/scripts/soak/publish_remote231.sh"
|
||||
install -m 0755 "$SCRIPT_DIR/publish_remote231_full.sh" "$STAGE_DIR/scripts/soak/publish_remote231_full.sh"
|
||||
install -m 0755 "$REPO_ROOT/scripts/verification/run_verification_only.sh" "$STAGE_DIR/scripts/verification/run_verification_only.sh"
|
||||
install -m 0644 "$REPO_ROOT/scripts/verification/README.md" "$STAGE_DIR/scripts/verification/README.md"
|
||||
install -m 0644 "$SCRIPT_DIR/portable-soak.env.example" "$STAGE_DIR/.env"
|
||||
install -m 0644 "$SCRIPT_DIR/portable-soak.env.example" "$STAGE_DIR/portable-soak.env.example"
|
||||
|
||||
|
||||
@@ -48,6 +48,7 @@ TMP_DIR="${TMP_DIR:-$RUN_ROOT/tmp}"
|
||||
RSYNC_MIRROR_ROOT="${RSYNC_MIRROR_ROOT:-$STATE_ROOT/rsync-mirror}"
|
||||
INVALID_ROOT="$STATE_ROOT/invalid"
|
||||
RESET_STAGING_ROOT="$STATE_ROOT/reset-staging"
|
||||
STATE_EXECUTION_LOCK_PATH="$STATE_ROOT/locks/state-execution.lock"
|
||||
LIVE_TA_REFRESH_DIR="${LIVE_TA_REFRESH_DIR:-$META_DIR/live-ta-refresh}"
|
||||
LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS="${LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS:-15}"
|
||||
LIVE_TA_REFRESH_MAX_TIME_SECS="${LIVE_TA_REFRESH_MAX_TIME_SECS:-120}"
|
||||
@@ -1080,6 +1081,7 @@ build_child_args() {
|
||||
|
||||
CHILD_ARGS+=(
|
||||
--report-json "{run_out}/report.json"
|
||||
--validation-contract-out "{run_out}/validation-contract.json"
|
||||
)
|
||||
if is_true "$OUTPUT_COMPACT_REPORT"; then
|
||||
CHILD_ARGS+=(--report-json-compact)
|
||||
@@ -1109,6 +1111,59 @@ build_child_args() {
|
||||
fi
|
||||
}
|
||||
|
||||
write_current_state_binding() {
|
||||
local run_dir="$1"
|
||||
local run_id="$2"
|
||||
local output_path="$META_DIR/current-state-binding.json"
|
||||
python3 - "$run_dir" "$run_id" "$output_path" <<'PY'
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
run_dir = Path(sys.argv[1])
|
||||
run_id = sys.argv[2]
|
||||
output_path = Path(sys.argv[3])
|
||||
summary = json.loads((run_dir / "run-summary.json").read_text(encoding="utf-8"))
|
||||
if summary.get("status") != "success":
|
||||
raise SystemExit(f"cannot bind unsuccessful source run: {run_id}")
|
||||
|
||||
def digest(name):
|
||||
path = run_dir / name
|
||||
if not path.is_file():
|
||||
raise SystemExit(f"current state binding input is missing: {path}")
|
||||
hasher = hashlib.sha256()
|
||||
with path.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
binding = {
|
||||
"schemaVersion": 1,
|
||||
"currentRunId": run_id,
|
||||
"sourceCirSha256": digest("input.cir"),
|
||||
"sourceCcrSha256": digest("result.ccr"),
|
||||
"validationContractSha256": digest("validation-contract.json"),
|
||||
"updatedAtRfc3339Utc": datetime.now(timezone.utc).isoformat().replace("+00:00", "Z"),
|
||||
}
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
fd, tmp_name = tempfile.mkstemp(prefix=output_path.name + ".", dir=output_path.parent)
|
||||
try:
|
||||
with os.fdopen(fd, "w", encoding="utf-8") as handle:
|
||||
json.dump(binding, handle, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
handle.flush()
|
||||
os.fsync(handle.fileno())
|
||||
os.replace(tmp_name, output_path)
|
||||
finally:
|
||||
if os.path.exists(tmp_name):
|
||||
os.unlink(tmp_name)
|
||||
PY
|
||||
}
|
||||
|
||||
copy_inner_run_outputs() {
|
||||
local daemon_state_root="$1"
|
||||
local run_dir="$2"
|
||||
@@ -1304,6 +1359,12 @@ run_one_round() {
|
||||
final_status="failed"
|
||||
fi
|
||||
fi
|
||||
if [[ "$final_status" == "success" ]]; then
|
||||
if ! write_current_state_binding "$run_dir" "$run_id"; then
|
||||
warn "failed to bind successful run to current state: $run_id"
|
||||
final_status="failed"
|
||||
fi
|
||||
fi
|
||||
write_run_meta "$run_dir/run-meta.json" "$final_status" "$run_index" "$run_id" "$sync_mode" \
|
||||
"$snapshot_reason" "$previous_run_id" "$previous_success_value" "$started_at" "$completed_at" \
|
||||
"$INVALID_DB_PATH" "$INVALID_STATE_PATH" "$INVALID_TMP_PATH" "$daemon_exit_code" "$PACKAGE_ROOT" "$ENV_FILE" \
|
||||
@@ -1327,6 +1388,7 @@ main() {
|
||||
require_command python3
|
||||
require_command date
|
||||
require_command find
|
||||
require_command flock
|
||||
if [[ "$TAL_INPUT_MODE" == "file-live-ta" ]]; then
|
||||
require_command curl
|
||||
validate_positive_int "LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS" "$LIVE_TA_REFRESH_CONNECT_TIMEOUT_SECS"
|
||||
@@ -1360,7 +1422,7 @@ main() {
|
||||
fi
|
||||
done
|
||||
|
||||
mkdir -p "$RUNS_ROOT" "$LOG_ROOT" "$DB_DIR" "$META_DIR" "$TMP_DIR" "$INVALID_ROOT" "$RESET_STAGING_ROOT" "$LIVE_TA_REFRESH_DIR"
|
||||
mkdir -p "$RUNS_ROOT" "$LOG_ROOT" "$DB_DIR" "$META_DIR" "$TMP_DIR" "$INVALID_ROOT" "$RESET_STAGING_ROOT" "$LIVE_TA_REFRESH_DIR" "$(dirname "$STATE_EXECUTION_LOCK_PATH")"
|
||||
if is_true "$ALLOW_RSYNC_MIRROR_REUSE"; then
|
||||
mkdir -p "$RSYNC_MIRROR_ROOT"
|
||||
fi
|
||||
@@ -1386,6 +1448,9 @@ main() {
|
||||
local any_failed=0
|
||||
|
||||
while (( run_forever == 1 || next_index <= stop_index )); do
|
||||
local state_lock_fd
|
||||
exec {state_lock_fd}>"$STATE_EXECUTION_LOCK_PATH"
|
||||
flock "$state_lock_fd"
|
||||
INVALID_DB_PATH=""
|
||||
INVALID_STATE_PATH=""
|
||||
INVALID_TMP_PATH=""
|
||||
@@ -1472,6 +1537,8 @@ main() {
|
||||
echo "completed run $(printf 'run_%04d' "$next_index") status=failed" >&2
|
||||
any_failed=1
|
||||
fi
|
||||
flock -u "$state_lock_fd"
|
||||
exec {state_lock_fd}>&-
|
||||
if (( STOP_AFTER_SECS > 0 )); then
|
||||
elapsed_secs=$(( $(date +%s) - started_epoch ))
|
||||
if (( elapsed_secs >= STOP_AFTER_SECS )); then
|
||||
|
||||
Reference in New Issue
Block a user