564 lines
28 KiB
Bash
Executable File
564 lines
28 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Observe a Docker-managed soak from the host without joining its compose project.
|
|
# The verifier binary must be copied byte-for-byte from the running runtime image;
|
|
# validation-contract.json rejects a different binary SHA-256.
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage:
|
|
observe_remote231_verification_only.sh \
|
|
--runs-root <normal_runs_dir> \
|
|
--state-root <normal_state_dir> \
|
|
--rpki-bin <host_side_rpki> \
|
|
--work-root <observer_root> \
|
|
--webhook-url-file <0600_file> \
|
|
[--target-runs 20] [--delay-secs 30] [--poll-secs 5] \
|
|
[--verification-timeout-secs 480] [--normal-interval-secs 600] \
|
|
[--normal-run-safety-secs 60] [--verification-object-workers 4] \
|
|
[--verification-child-path <path>]
|
|
|
|
This is a host-side, finite observer. On its first start it records the latest
|
|
already-terminal normal run and observes only later terminal runs. For each one
|
|
it waits DELAY seconds, then uses a non-blocking state lock. It never waits for
|
|
or blocks the Docker soak. Successful normal runs are verified with a copied
|
|
host-side rpki binary; failed, stale, or lock-busy runs are recorded as skipped.
|
|
|
|
The verification timeout is additionally bounded by the source run completion
|
|
time plus NORMAL_INTERVAL_SECS minus NORMAL_RUN_SAFETY_SECS. This preserves a
|
|
normal-run safety window even after an observer restart or delayed scan.
|
|
|
|
Only compact JSON results are retained under WORK_ROOT/results/. The large
|
|
verification output and any retained scratch DB are always deleted afterwards.
|
|
|
|
When --verification-child-path is supplied, it replaces PATH only for the
|
|
verification binary. Passing a directory without rsync (for example
|
|
/nonexistent) is an adversarial acceptance check: an accidental SystemRsync
|
|
fallback must fail instead of accessing the network.
|
|
USAGE
|
|
}
|
|
|
|
RUNS_ROOT=""
|
|
STATE_ROOT=""
|
|
RPKI_BIN=""
|
|
WORK_ROOT=""
|
|
WEBHOOK_URL_FILE=""
|
|
TARGET_RUNS=20
|
|
DELAY_SECS=30
|
|
POLL_SECS=5
|
|
VERIFICATION_TIMEOUT_SECS=480
|
|
NORMAL_INTERVAL_SECS=600
|
|
NORMAL_RUN_SAFETY_SECS=60
|
|
VERIFICATION_OBJECT_WORKERS=4
|
|
VERIFICATION_CHILD_PATH=""
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--runs-root) RUNS_ROOT="${2:?missing value}"; shift 2 ;;
|
|
--state-root) STATE_ROOT="${2:?missing value}"; shift 2 ;;
|
|
--rpki-bin) RPKI_BIN="${2:?missing value}"; shift 2 ;;
|
|
--work-root) WORK_ROOT="${2:?missing value}"; shift 2 ;;
|
|
--webhook-url-file) WEBHOOK_URL_FILE="${2:?missing value}"; shift 2 ;;
|
|
--target-runs) TARGET_RUNS="${2:?missing value}"; shift 2 ;;
|
|
--delay-secs) DELAY_SECS="${2:?missing value}"; shift 2 ;;
|
|
--poll-secs) POLL_SECS="${2:?missing value}"; shift 2 ;;
|
|
--verification-timeout-secs) VERIFICATION_TIMEOUT_SECS="${2:?missing value}"; shift 2 ;;
|
|
--normal-interval-secs) NORMAL_INTERVAL_SECS="${2:?missing value}"; shift 2 ;;
|
|
--normal-run-safety-secs) NORMAL_RUN_SAFETY_SECS="${2:?missing value}"; shift 2 ;;
|
|
--verification-object-workers) VERIFICATION_OBJECT_WORKERS="${2:?missing value}"; shift 2 ;;
|
|
--verification-child-path) VERIFICATION_CHILD_PATH="${2:?missing value}"; shift 2 ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
for value_name in RUNS_ROOT STATE_ROOT RPKI_BIN WORK_ROOT WEBHOOK_URL_FILE; do
|
|
[[ -n "${!value_name}" ]] || { echo "$value_name is required" >&2; exit 2; }
|
|
done
|
|
for integer_name in TARGET_RUNS DELAY_SECS POLL_SECS VERIFICATION_TIMEOUT_SECS NORMAL_INTERVAL_SECS VERIFICATION_OBJECT_WORKERS; do
|
|
[[ "${!integer_name}" =~ ^[0-9]+$ ]] && [[ "${!integer_name}" != 0 ]] || {
|
|
echo "$integer_name must be a positive integer" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
[[ "$NORMAL_RUN_SAFETY_SECS" =~ ^[0-9]+$ ]] || {
|
|
echo "NORMAL_RUN_SAFETY_SECS must be a non-negative integer" >&2
|
|
exit 2
|
|
}
|
|
(( NORMAL_RUN_SAFETY_SECS < NORMAL_INTERVAL_SECS )) || {
|
|
echo "NORMAL_RUN_SAFETY_SECS must be smaller than NORMAL_INTERVAL_SECS" >&2
|
|
exit 2
|
|
}
|
|
[[ -d "$RUNS_ROOT" ]] || { echo "runs root is not a directory: $RUNS_ROOT" >&2; exit 2; }
|
|
[[ -d "$STATE_ROOT" ]] || { echo "state root is not a directory: $STATE_ROOT" >&2; exit 2; }
|
|
[[ -x "$RPKI_BIN" ]] || { echo "rpki binary is not executable: $RPKI_BIN" >&2; exit 2; }
|
|
[[ -f "$WEBHOOK_URL_FILE" ]] || { echo "webhook URL file is absent: $WEBHOOK_URL_FILE" >&2; exit 2; }
|
|
for required_command in curl flock ionice jq nice python3 sha256sum timeout; do
|
|
command -v "$required_command" >/dev/null || {
|
|
echo "missing required command: $required_command" >&2
|
|
exit 2
|
|
}
|
|
done
|
|
|
|
mkdir -p "$WORK_ROOT/results" "$WORK_ROOT/scratch"
|
|
STATE_FILE="$WORK_ROOT/observer-state.json"
|
|
OBSERVER_LOCK="$WORK_ROOT/observer.lock"
|
|
RESULTS_DIR="$WORK_ROOT/results"
|
|
SCRATCH_ROOT="$WORK_ROOT/scratch"
|
|
WEBHOOK_URL="$(tr -d '\r\n' < "$WEBHOOK_URL_FILE")"
|
|
[[ "$WEBHOOK_URL" == https://open.feishu.cn/open-apis/bot/v2/hook/* ]] || {
|
|
echo "webhook URL does not have the expected Feishu incoming-webhook form" >&2
|
|
exit 2
|
|
}
|
|
|
|
now_utc() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
|
now_ms() { python3 -c 'import time; print(time.time_ns() // 1_000_000)'; }
|
|
|
|
remaining_verification_window_secs() {
|
|
local completed_at="$1"
|
|
python3 - "$completed_at" "$NORMAL_INTERVAL_SECS" "$NORMAL_RUN_SAFETY_SECS" <<'PY'
|
|
import datetime
|
|
import sys
|
|
import time
|
|
|
|
completed = datetime.datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00"))
|
|
interval = int(sys.argv[2])
|
|
safety = int(sys.argv[3])
|
|
print(int(completed.timestamp()) + interval - safety - int(time.time()))
|
|
PY
|
|
}
|
|
|
|
atomic_replace() {
|
|
local destination="$1"
|
|
local temporary="${destination}.tmp.$$"
|
|
cat > "$temporary"
|
|
mv "$temporary" "$destination"
|
|
}
|
|
|
|
terminal_run_ids() {
|
|
local run_dir meta
|
|
for run_dir in "$RUNS_ROOT"/run_[0-9]*; do
|
|
[[ -d "$run_dir" ]] || continue
|
|
meta="$run_dir/run-meta.json"
|
|
[[ -f "$meta" ]] || continue
|
|
jq -e '(.completed_at_rfc3339_utc? // .completedAt? // null) != null' "$meta" >/dev/null 2>&1 || continue
|
|
basename "$run_dir"
|
|
done | sort -t_ -k2,2n
|
|
}
|
|
|
|
run_number() {
|
|
local run_id="$1"
|
|
printf '%d\n' "$((10#${run_id#run_}))"
|
|
}
|
|
|
|
latest_terminal_run_id() {
|
|
terminal_run_ids | tail -n 1
|
|
}
|
|
|
|
result_path() {
|
|
printf '%s/%s.json\n' "$RESULTS_DIR" "$1"
|
|
}
|
|
|
|
read_json_or_empty() {
|
|
local path="$1"
|
|
if [[ -f "$path" ]] && jq -e . "$path" >/dev/null 2>&1; then
|
|
jq -c . "$path"
|
|
else
|
|
printf '{}'
|
|
fi
|
|
}
|
|
|
|
update_cumulative() {
|
|
local observed normal_success passes mismatches errors skipped pending latest
|
|
local -a result_files
|
|
shopt -s nullglob
|
|
result_files=("$RESULTS_DIR"/run_*.json)
|
|
shopt -u nullglob
|
|
observed="${#result_files[@]}"
|
|
if (( observed == 0 )); then
|
|
normal_success=0
|
|
passes=0
|
|
mismatches=0
|
|
errors=0
|
|
skipped=0
|
|
pending=0
|
|
else
|
|
normal_success="$(jq -r 'select(.normalRun.status == "success") | 1' "${result_files[@]}" | wc -l)"
|
|
passes="$(jq -r 'select(.verification.status == "pass") | 1' "${result_files[@]}" | wc -l)"
|
|
mismatches="$(jq -r 'select(.verification.status == "mismatch") | 1' "${result_files[@]}" | wc -l)"
|
|
errors="$(jq -r 'select(.verification.status == "error") | 1' "${result_files[@]}" | wc -l)"
|
|
skipped="$(jq -r 'select(.verification.status | startswith("skipped_")) | 1' "${result_files[@]}" | wc -l)"
|
|
pending="$(jq -r 'select(.notification.status != "sent") | 1' "${result_files[@]}" | wc -l)"
|
|
fi
|
|
latest="$(find "$RESULTS_DIR" -maxdepth 1 -type f -name 'run_*.json' -printf '%f\n' | sed 's/.json$//' | sort -t_ -k2,2n | tail -n 1)"
|
|
jq -n \
|
|
--arg updated_at "$(now_utc)" \
|
|
--arg initial_run_id "$(jq -r '.initialLatestTerminalRunId' "$STATE_FILE")" \
|
|
--arg latest_run_id "$latest" \
|
|
--argjson target_runs "$TARGET_RUNS" \
|
|
--argjson observed_runs "$observed" \
|
|
--argjson normal_success_runs "$normal_success" \
|
|
--argjson verification_passes "$passes" \
|
|
--argjson verification_mismatches "$mismatches" \
|
|
--argjson verification_errors "$errors" \
|
|
--argjson verification_skipped "$skipped" \
|
|
--argjson notification_pending "$pending" \
|
|
'{schemaVersion: 1, updatedAt: $updated_at, initialLatestTerminalRunId: $initial_run_id,
|
|
latestProcessedRunId: $latest_run_id, targetRuns: $target_runs, observedRuns: $observed_runs,
|
|
normalSuccessRuns: $normal_success_runs, verificationPasses: $verification_passes,
|
|
verificationMismatches: $verification_mismatches, verificationErrors: $verification_errors,
|
|
verificationSkipped: $verification_skipped, notificationPending: $notification_pending}' \
|
|
| atomic_replace "$RESULTS_DIR/cumulative.json"
|
|
}
|
|
|
|
notify_result() {
|
|
local path="$1" run_id status observed target passes mismatches errors skipped analysis_available top_phase message payload response
|
|
run_id="$(jq -r '.runId' "$path")"
|
|
status="$(jq -r '.verification.status' "$path")"
|
|
observed="$(jq -r '.observedRuns' "$RESULTS_DIR/cumulative.json")"
|
|
target="$(jq -r '.targetRuns' "$RESULTS_DIR/cumulative.json")"
|
|
passes="$(jq -r '.verificationPasses' "$RESULTS_DIR/cumulative.json")"
|
|
mismatches="$(jq -r '.verificationMismatches' "$RESULTS_DIR/cumulative.json")"
|
|
errors="$(jq -r '.verificationErrors' "$RESULTS_DIR/cumulative.json")"
|
|
skipped="$(jq -r '.verificationSkipped' "$RESULTS_DIR/cumulative.json")"
|
|
analysis_available="$(jq -r '.verification.analysis.available // false' "$path")"
|
|
top_phase="$(jq -r '(.verification.analysis.phaseTotals // [])[0] | if . == null then "n/a" else "\(.name)=\(.totalMs)ms" end' "$path")"
|
|
message="From codex: #137 远端231 verification-only 旁路复核\n${run_id}: ${status}\n"
|
|
message+="CCR state digest match=$(jq -r 'if .verification.comparison | has("stateDigestMatch") then (.verification.comparison.stateDigestMatch | tostring) else "n/a" end' "$path"); fallback replay match=$(jq -r 'if .verification.comparison.fallbackReplaySelection | has("matches") then (.verification.comparison.fallbackReplaySelection.matches | tostring) else "n/a" end' "$path"); verificationProcessMs=$(jq -r '.verification.timing.verificationProcessMs // .verification.wallMs // "n/a"' "$path"); preValidationPreparationMs=$(jq -r '.verification.timing.preValidationPreparationMs // "n/a"' "$path"); lockWaitMs=$(jq -r '.verification.lockWaitMs // "n/a"' "$path")\n"
|
|
message+="analysisAvailable=${analysis_available}; topPhase=${top_phase}\n"
|
|
message+="累计 ${observed}/${target}: pass=${passes}, mismatch=${mismatches}, error=${errors}, skipped=${skipped}\n"
|
|
message+="结果: ${path}"
|
|
payload="$(jq -n --arg text "$message" '{msg_type: "text", content: {text: $text}}')"
|
|
if response="$(curl --fail-with-body --silent --show-error --connect-timeout 10 --max-time 30 \
|
|
-H 'Content-Type: application/json' --data "$payload" "$WEBHOOK_URL" 2>&1)"; then
|
|
if jq -e '((.code // .StatusCode // 0) == 0)' <<< "$response" >/dev/null 2>&1; then
|
|
jq --arg sent_at "$(now_utc)" --arg response "$response" \
|
|
'.notification = {status: "sent", sentAt: $sent_at, response: $response}' "$path" \
|
|
| atomic_replace "$path"
|
|
return 0
|
|
fi
|
|
response="Feishu rejected payload: $response"
|
|
fi
|
|
jq --arg attempted_at "$(now_utc)" --arg error "$response" \
|
|
'.notification = {status: "pending", lastAttemptAt: $attempted_at, lastError: $error}' "$path" \
|
|
| atomic_replace "$path"
|
|
return 1
|
|
}
|
|
|
|
deliver_pending_notifications() {
|
|
local path
|
|
for path in "$RESULTS_DIR"/run_*.json; do
|
|
[[ -f "$path" ]] || continue
|
|
[[ "$(jq -r '.notification.status // "pending"' "$path")" == "sent" ]] && continue
|
|
update_cumulative
|
|
notify_result "$path" || true
|
|
done
|
|
update_cumulative
|
|
}
|
|
|
|
write_result() {
|
|
local run_id="$1" normal_json="$2" verification_json="$3" result
|
|
result="$(result_path "$run_id")"
|
|
jq -n \
|
|
--arg recorded_at "$(now_utc)" \
|
|
--arg run_id "$run_id" \
|
|
--argjson normal_run "$normal_json" \
|
|
--argjson verification "$verification_json" \
|
|
'{schemaVersion: 1, recordedAt: $recorded_at, runId: $run_id, normalRun: $normal_run,
|
|
verification: $verification, notification: {status: "pending"}}' \
|
|
| atomic_replace "$result"
|
|
update_cumulative
|
|
notify_result "$result" || true
|
|
update_cumulative
|
|
}
|
|
|
|
normal_run_summary() {
|
|
local run_dir="$1"
|
|
if [[ ! -f "$run_dir/run-meta.json" ]]; then
|
|
printf '{"status":"invalid-meta"}'
|
|
return
|
|
fi
|
|
jq -cn --slurpfile meta "$run_dir/run-meta.json" --slurpfile summary "$run_dir/run-summary.json" \
|
|
'{status: ($meta[0].status // "unknown"), syncMode: ($meta[0].sync_mode // $meta[0].syncMode // "unknown"),
|
|
runIndex: ($meta[0].run_index // $meta[0].runIndex // null),
|
|
completedAt: ($meta[0].completed_at_rfc3339_utc // $meta[0].completedAt // null),
|
|
wallMs: ($summary[0].wall_ms // $summary[0].wallMs // null),
|
|
exitCode: ($summary[0].exit_code // $summary[0].exitCode // $meta[0].daemon_exit_code // null)}' 2>/dev/null \
|
|
|| printf '{"status":"invalid-meta"}'
|
|
}
|
|
|
|
process_terminal_run() {
|
|
local run_id="$1" run_dir="$RUNS_ROOT/$run_id" normal_json status temporary_dir state_lock_fd lock_started_ms lock_acquired_ms
|
|
local binding_run_id binary_sha source_sha run_started_ms run_finished_ms exit_code comparison_json comparison_state_digest_match fallback_replay_comparison_json fallback_replay_selection_match verification_meta_json stage_timing_json analysis_json analysis_summary_json error_tail max_rss_kb verification_status verification_json timeout_secs completed_at
|
|
local -a verification_child_exec
|
|
local observer_started_ms delay_finished_ms preparation_started_ms binding_started_ms binding_finished_ms binary_check_started_ms binary_check_finished_ms window_started_ms window_finished_ms scratch_cleanup_started_ms scratch_cleanup_finished_ms post_validation_started_ms post_validation_collection_finished_ms result_cleanup_started_ms result_cleanup_finished_ms
|
|
normal_json="$(normal_run_summary "$run_dir")"
|
|
status="$(jq -r '.status' <<< "$normal_json")"
|
|
observer_started_ms="$(now_ms)"
|
|
sleep "$DELAY_SECS"
|
|
delay_finished_ms="$(now_ms)"
|
|
if [[ "$status" != "success" ]]; then
|
|
write_result "$run_id" "$normal_json" '{"status":"skipped_normal_run_not_success"}'
|
|
return
|
|
fi
|
|
|
|
preparation_started_ms="$delay_finished_ms"
|
|
mkdir -p "$STATE_ROOT/locks"
|
|
exec {state_lock_fd}>"$STATE_ROOT/locks/state-execution.lock"
|
|
lock_started_ms="$(now_ms)"
|
|
if ! flock -n "$state_lock_fd"; then
|
|
exec {state_lock_fd}>&-
|
|
write_result "$run_id" "$normal_json" '{"status":"skipped_state_lock_busy"}'
|
|
return
|
|
fi
|
|
lock_acquired_ms="$(now_ms)"
|
|
binding_started_ms="$(now_ms)"
|
|
binding_run_id="$(jq -r '.currentRunId // empty' "$STATE_ROOT/meta/current-state-binding.json" 2>/dev/null || true)"
|
|
binding_finished_ms="$(now_ms)"
|
|
if [[ "$binding_run_id" != "$run_id" ]]; then
|
|
flock -u "$state_lock_fd"
|
|
exec {state_lock_fd}>&-
|
|
verification_json="$(jq -n --arg bound_run "$binding_run_id" --argjson lock_wait_ms "$((lock_acquired_ms - lock_started_ms))" \
|
|
'{status: "skipped_source_no_longer_current", boundRunId: $bound_run, lockWaitMs: $lock_wait_ms}')"
|
|
write_result "$run_id" "$normal_json" "$verification_json"
|
|
return
|
|
fi
|
|
binary_check_started_ms="$(now_ms)"
|
|
source_sha="$(jq -r '.binarySha256 // empty' "$run_dir/validation-contract.json" 2>/dev/null || true)"
|
|
binary_sha="$(sha256sum "$RPKI_BIN" | awk '{print $1}')"
|
|
binary_check_finished_ms="$(now_ms)"
|
|
if [[ -z "$source_sha" || "$source_sha" != "$binary_sha" ]]; then
|
|
flock -u "$state_lock_fd"
|
|
exec {state_lock_fd}>&-
|
|
verification_json="$(jq -n --arg expected "$source_sha" --arg actual "$binary_sha" --argjson lock_wait_ms "$((lock_acquired_ms - lock_started_ms))" \
|
|
'{status: "error_binary_sha256_mismatch", expectedBinarySha256: $expected, actualBinarySha256: $actual, lockWaitMs: $lock_wait_ms}')"
|
|
write_result "$run_id" "$normal_json" "$verification_json"
|
|
return
|
|
fi
|
|
|
|
window_started_ms="$(now_ms)"
|
|
completed_at="$(jq -r '.completedAt // empty' <<< "$normal_json")"
|
|
timeout_secs="$(remaining_verification_window_secs "$completed_at")"
|
|
window_finished_ms="$(now_ms)"
|
|
if (( timeout_secs <= 0 )); then
|
|
flock -u "$state_lock_fd"
|
|
exec {state_lock_fd}>&-
|
|
verification_json="$(jq -n --arg completed_at "$completed_at" --argjson lock_wait_ms "$((lock_acquired_ms - lock_started_ms))" \
|
|
'{status: "skipped_insufficient_interval", sourceCompletedAt: $completed_at, lockWaitMs: $lock_wait_ms}')"
|
|
write_result "$run_id" "$normal_json" "$verification_json"
|
|
return
|
|
fi
|
|
if (( timeout_secs > VERIFICATION_TIMEOUT_SECS )); then
|
|
timeout_secs="$VERIFICATION_TIMEOUT_SECS"
|
|
fi
|
|
|
|
temporary_dir="$SCRATCH_ROOT/$run_id"
|
|
scratch_cleanup_started_ms="$(now_ms)"
|
|
rm -rf "$temporary_dir"
|
|
scratch_cleanup_finished_ms="$(now_ms)"
|
|
run_started_ms="$(now_ms)"
|
|
verification_child_exec=()
|
|
if [[ -n "$VERIFICATION_CHILD_PATH" ]]; then
|
|
verification_child_exec=(/usr/bin/env "PATH=$VERIFICATION_CHILD_PATH")
|
|
fi
|
|
set +e
|
|
RPKI_STATE_EXECUTION_LOCK_HELD=1 /usr/bin/time -v -o "$temporary_dir.process-time" -- \
|
|
/usr/bin/timeout --foreground --signal=TERM --kill-after=30s "$timeout_secs" \
|
|
/usr/bin/ionice -c2 -n0 "${verification_child_exec[@]}" "$RPKI_BIN" \
|
|
--verification-only --verification-source-run "$run_dir" \
|
|
--verification-source-state "$STATE_ROOT" --verification-out "$temporary_dir" \
|
|
--analyze --analysis-out "$temporary_dir/analyze" \
|
|
--parallel-phase2-object-workers "$VERIFICATION_OBJECT_WORKERS" \
|
|
>"$temporary_dir.stdout" 2>"$temporary_dir.stderr"
|
|
exit_code=$?
|
|
set -e
|
|
run_finished_ms="$(now_ms)"
|
|
post_validation_started_ms="$run_finished_ms"
|
|
flock -u "$state_lock_fd"
|
|
exec {state_lock_fd}>&-
|
|
|
|
comparison_json="$(read_json_or_empty "$temporary_dir/compare/ccr-state-digest.json")"
|
|
fallback_replay_comparison_json="$(read_json_or_empty "$temporary_dir/compare/fallback-replay-selection.json")"
|
|
verification_meta_json="$(read_json_or_empty "$temporary_dir/verification-meta.json")"
|
|
stage_timing_json="$(read_json_or_empty "$temporary_dir/stage-timing.json")"
|
|
analysis_json="$(read_json_or_empty "$temporary_dir/analyze/timing.json")"
|
|
analysis_summary_json="$(jq -cn --argjson stage "$stage_timing_json" --argjson analysis "$analysis_json" '
|
|
{enabled: true,
|
|
available: (($analysis | type) == "object" and ($analysis | length) > 0),
|
|
stageTiming: {
|
|
totalMs: ($stage.total_ms // null),
|
|
validationMs: ($stage.validation_ms // null),
|
|
repoSyncMsTotal: ($stage.repo_sync_ms_total // null),
|
|
publicationPointRepoSyncMsTotal: ($stage.publication_point_repo_sync_ms_total // null),
|
|
rrdpDownloadMsTotal: ($stage.rrdp_download_ms_total // null),
|
|
rsyncDownloadMsTotal: ($stage.rsync_download_ms_total // null),
|
|
reportBuildMs: ($stage.report_build_ms // null),
|
|
reportWriteMs: ($stage.report_write_ms // null),
|
|
ccrBuildMs: ($stage.ccr_build_ms // null),
|
|
ccrWriteMs: ($stage.ccr_write_ms // null),
|
|
cirTotalMs: ($stage.cir_total_ms // null)
|
|
},
|
|
phaseTotals: (($analysis.phases // {})
|
|
| to_entries
|
|
| map({name: .key, count: (.value.count // 0), totalMs: (((.value.total_nanos // 0) / 1000000) | floor)})
|
|
| sort_by(-.totalMs)
|
|
| .[0:20]),
|
|
topPublicationPoints: (($analysis.top_publication_points // [])[0:10]),
|
|
topPublicationPointSteps: (($analysis.top_publication_point_steps // [])[0:10]),
|
|
topRrdpRepos: (($analysis.top_rrdp_repos // [])[0:10]),
|
|
topRrdpRepoSteps: (($analysis.top_rrdp_repo_steps // [])[0:10])}')"
|
|
error_tail="$(tail -n 20 "$temporary_dir.stderr" 2>/dev/null || true)"
|
|
max_rss_kb="$(awk -F: '/Maximum resident set size/{gsub(/^[[:space:]]+/, "", $2); print $2}' "$temporary_dir.process-time" 2>/dev/null || true)"
|
|
comparison_state_digest_match="$(jq -r 'if has("stateDigestMatch") then (.stateDigestMatch | tostring) else "missing" end' <<< "$comparison_json")"
|
|
fallback_replay_selection_match="$(jq -r 'if has("matches") then (.matches | tostring) else "missing" end' <<< "$fallback_replay_comparison_json")"
|
|
if [[ "$comparison_state_digest_match" == "true" && "$fallback_replay_selection_match" == "true" && "$exit_code" -eq 0 ]]; then
|
|
verification_status="pass"
|
|
elif [[ "$comparison_state_digest_match" == "false" || "$fallback_replay_selection_match" == "false" ]]; then
|
|
verification_status="mismatch"
|
|
else
|
|
verification_status="error"
|
|
fi
|
|
post_validation_collection_finished_ms="$(now_ms)"
|
|
result_cleanup_started_ms="$(now_ms)"
|
|
rm -rf "$temporary_dir" "$temporary_dir.stdout" "$temporary_dir.stderr" "$temporary_dir.process-time"
|
|
result_cleanup_finished_ms="$(now_ms)"
|
|
verification_json="$(jq -n \
|
|
--arg status "$verification_status" \
|
|
--arg verification_child_path "$VERIFICATION_CHILD_PATH" \
|
|
--argjson exit_code "$exit_code" \
|
|
--argjson wall_ms "$((run_finished_ms - run_started_ms))" \
|
|
--argjson lock_wait_ms "$((lock_acquired_ms - lock_started_ms))" \
|
|
--argjson timeout_limit_secs "$timeout_secs" \
|
|
--argjson object_workers "$VERIFICATION_OBJECT_WORKERS" \
|
|
--arg max_rss_kb "${max_rss_kb:-}" \
|
|
--arg error_tail "$error_tail" \
|
|
--argjson comparison "$comparison_json" \
|
|
--argjson fallback_replay_comparison "$fallback_replay_comparison_json" \
|
|
--argjson meta "$verification_meta_json" \
|
|
--argjson analysis "$analysis_summary_json" \
|
|
--argjson post_run_delay_ms "$((delay_finished_ms - observer_started_ms))" \
|
|
--argjson observer_to_verification_start_ms "$((run_started_ms - observer_started_ms))" \
|
|
--argjson pre_validation_preparation_ms "$((run_started_ms - preparation_started_ms))" \
|
|
--argjson binding_check_ms "$((binding_finished_ms - binding_started_ms))" \
|
|
--argjson binary_contract_check_ms "$((binary_check_finished_ms - binary_check_started_ms))" \
|
|
--argjson interval_window_calculation_ms "$((window_finished_ms - window_started_ms))" \
|
|
--argjson scratch_preparation_cleanup_ms "$((scratch_cleanup_finished_ms - scratch_cleanup_started_ms))" \
|
|
--argjson verification_process_ms "$((run_finished_ms - run_started_ms))" \
|
|
--argjson post_validation_collection_ms "$((post_validation_collection_finished_ms - post_validation_started_ms))" \
|
|
--argjson result_cleanup_ms "$((result_cleanup_finished_ms - result_cleanup_started_ms))" \
|
|
'{status: $status, exitCode: $exit_code, wallMs: $wall_ms, lockWaitMs: $lock_wait_ms,
|
|
timeoutLimitSecs: $timeout_limit_secs, objectWorkers: $object_workers,
|
|
verificationChildPath: (if $verification_child_path == "" then null else $verification_child_path end),
|
|
timing: {postRunDelayMs: $post_run_delay_ms,
|
|
observerToVerificationStartMs: $observer_to_verification_start_ms,
|
|
preValidationPreparationMs: $pre_validation_preparation_ms,
|
|
preparationBreakdownMs: {stateLockWaitMs: $lock_wait_ms,
|
|
bindingCheckMs: $binding_check_ms,
|
|
binaryContractCheckMs: $binary_contract_check_ms,
|
|
intervalWindowCalculationMs: $interval_window_calculation_ms,
|
|
scratchPreparationCleanupMs: $scratch_preparation_cleanup_ms},
|
|
verificationProcessMs: $verification_process_ms,
|
|
postValidationCollectionMs: $post_validation_collection_ms,
|
|
resultCleanupMs: $result_cleanup_ms},
|
|
maxRssKb: (if $max_rss_kb == "" then null else ($max_rss_kb | tonumber) end),
|
|
comparison: {stateDigestMatch: ($comparison | if has("stateDigestMatch") then .stateDigestMatch else null end),
|
|
mismatchedStates: ($comparison.mismatchedStates // []),
|
|
states: (($comparison.states // []) | map({name, sourcePresent, verificationPresent,
|
|
sourceHashHex, verificationHashHex, matches})),
|
|
fallbackReplaySelection: {
|
|
matches: ($fallback_replay_comparison | if has("matches") then .matches else null end),
|
|
expectedCount: (($fallback_replay_comparison.expected // []) | length),
|
|
actualCount: (($fallback_replay_comparison.actual // []) | length),
|
|
missing: ($fallback_replay_comparison.missing // []),
|
|
unexpected: ($fallback_replay_comparison.unexpected // [])
|
|
}},
|
|
verificationMeta: {status: ($meta.status // null), wallMs: ($meta.wallMs // null),
|
|
repositoryObjectsChecked: ($meta.repositoryObjectsChecked // null),
|
|
repositoryBytesChecked: ($meta.repositoryBytesChecked // null)},
|
|
analysis: $analysis,
|
|
errorTail: $error_tail}')"
|
|
write_result "$run_id" "$normal_json" "$verification_json"
|
|
}
|
|
|
|
exec {observer_lock_fd}>"$OBSERVER_LOCK"
|
|
if ! flock -n "$observer_lock_fd"; then
|
|
echo "another observer already owns $OBSERVER_LOCK" >&2
|
|
exit 3
|
|
fi
|
|
|
|
find "$SCRATCH_ROOT" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
|
|
|
|
if [[ ! -f "$STATE_FILE" ]]; then
|
|
initial_run_id="$(latest_terminal_run_id)"
|
|
[[ -n "$initial_run_id" ]] || { echo "no terminal normal run found under $RUNS_ROOT" >&2; exit 2; }
|
|
jq -n \
|
|
--arg started_at "$(now_utc)" \
|
|
--arg initial_run_id "$initial_run_id" \
|
|
--arg runs_root "$RUNS_ROOT" \
|
|
--arg state_root "$STATE_ROOT" \
|
|
--arg rpki_sha256 "$(sha256sum "$RPKI_BIN" | awk '{print $1}')" \
|
|
--argjson target_runs "$TARGET_RUNS" \
|
|
--argjson delay_secs "$DELAY_SECS" \
|
|
--argjson verification_timeout_secs "$VERIFICATION_TIMEOUT_SECS" \
|
|
--argjson normal_interval_secs "$NORMAL_INTERVAL_SECS" \
|
|
--argjson normal_run_safety_secs "$NORMAL_RUN_SAFETY_SECS" \
|
|
--argjson verification_object_workers "$VERIFICATION_OBJECT_WORKERS" \
|
|
--arg verification_child_path "$VERIFICATION_CHILD_PATH" \
|
|
'{schemaVersion: 1, startedAt: $started_at, initialLatestTerminalRunId: $initial_run_id,
|
|
runsRoot: $runs_root, stateRoot: $state_root, verifierBinarySha256: $rpki_sha256,
|
|
targetRuns: $target_runs, delaySecs: $delay_secs, verificationTimeoutSecs: $verification_timeout_secs,
|
|
normalIntervalSecs: $normal_interval_secs, normalRunSafetySecs: $normal_run_safety_secs,
|
|
verificationObjectWorkers: $verification_object_workers,
|
|
verificationChildPath: (if $verification_child_path == "" then null else $verification_child_path end),
|
|
verificationAnalyze: true}' \
|
|
| atomic_replace "$STATE_FILE"
|
|
update_cumulative
|
|
fi
|
|
|
|
# Preserve the original baseline run while recording the live configuration after
|
|
# an observer restart. This is particularly important for the normal-run safety
|
|
# window, whose length depends on the soak interval.
|
|
jq --arg updated_at "$(now_utc)" \
|
|
--argjson target_runs "$TARGET_RUNS" \
|
|
--argjson delay_secs "$DELAY_SECS" \
|
|
--argjson verification_timeout_secs "$VERIFICATION_TIMEOUT_SECS" \
|
|
--argjson normal_interval_secs "$NORMAL_INTERVAL_SECS" \
|
|
--argjson normal_run_safety_secs "$NORMAL_RUN_SAFETY_SECS" \
|
|
--argjson verification_object_workers "$VERIFICATION_OBJECT_WORKERS" \
|
|
--arg verification_child_path "$VERIFICATION_CHILD_PATH" \
|
|
'.updatedAt = $updated_at
|
|
| .targetRuns = $target_runs
|
|
| .delaySecs = $delay_secs
|
|
| .verificationTimeoutSecs = $verification_timeout_secs
|
|
| .normalIntervalSecs = $normal_interval_secs
|
|
| .normalRunSafetySecs = $normal_run_safety_secs
|
|
| .verificationObjectWorkers = $verification_object_workers
|
|
| .verificationChildPath = (if $verification_child_path == "" then null else $verification_child_path end)
|
|
| .verificationAnalyze = true' "$STATE_FILE" \
|
|
| atomic_replace "$STATE_FILE"
|
|
|
|
initial_run_number="$(run_number "$(jq -r '.initialLatestTerminalRunId' "$STATE_FILE")")"
|
|
while true; do
|
|
deliver_pending_notifications
|
|
observed_count="$(find "$RESULTS_DIR" -maxdepth 1 -type f -name 'run_*.json' -printf '%f\n' | wc -l)"
|
|
pending_count="$(jq -r '.notificationPending' "$RESULTS_DIR/cumulative.json")"
|
|
if (( observed_count >= TARGET_RUNS && pending_count == 0 )); then
|
|
break
|
|
fi
|
|
if (( observed_count < TARGET_RUNS )); then
|
|
while IFS= read -r run_id; do
|
|
[[ -n "$run_id" ]] || continue
|
|
(( $(run_number "$run_id") > initial_run_number )) || continue
|
|
[[ -f "$(result_path "$run_id")" ]] && continue
|
|
process_terminal_run "$run_id"
|
|
observed_count="$(find "$RESULTS_DIR" -maxdepth 1 -type f -name 'run_*.json' -printf '%f\n' | wc -l)"
|
|
(( observed_count < TARGET_RUNS )) || break
|
|
done < <(terminal_run_ids)
|
|
fi
|
|
sleep "$POLL_SECS"
|
|
done
|
|
|
|
update_cumulative
|
|
echo "completed $(jq -r '.observedRuns' "$RESULTS_DIR/cumulative.json") observed runs; results: $RESULTS_DIR/cumulative.json"
|