#!/usr/bin/env bash
set -euo pipefail

# Observe a Docker-managed soak from the host without joining its compose project.
# The verifier binary must be copied byte-for-byte from the running runtime image;
# validation-contract.json rejects a different binary SHA-256.

usage() {
  cat <<'USAGE'
Usage:
  observe_remote231_verification_only.sh \
    --runs-root <normal_runs_dir> \
    --state-root <normal_state_dir> \
    --rpki-bin <host_side_rpki> \
    --work-root <observer_root> \
    --webhook-url-file <0600_file> \
    [--target-runs 20] [--delay-secs 30] [--poll-secs 5] \
    [--verification-timeout-secs 480] [--normal-interval-secs 600] \
    [--normal-run-safety-secs 60] [--verification-object-workers 4] \
    [--verification-child-path <path>]

This is a host-side, finite observer. On its first start it records the latest
already-terminal normal run and observes only later terminal runs. For each one
it waits DELAY seconds, then uses a non-blocking state lock. It never waits for
or blocks the Docker soak. Successful normal runs are verified with a copied
host-side rpki binary; failed, stale, or lock-busy runs are recorded as skipped.

The verification timeout is additionally bounded by the source run completion
time plus NORMAL_INTERVAL_SECS minus NORMAL_RUN_SAFETY_SECS. This preserves a
normal-run safety window even after an observer restart or delayed scan.

Only compact JSON results are retained under WORK_ROOT/results/. The large
verification output and any retained scratch DB are always deleted afterwards.

When --verification-child-path is supplied, it replaces PATH only for the
verification binary. Passing a directory without rsync (for example
/nonexistent) is an adversarial acceptance check: an accidental SystemRsync
fallback must fail instead of accessing the network.
USAGE
}

RUNS_ROOT=""
STATE_ROOT=""
RPKI_BIN=""
WORK_ROOT=""
WEBHOOK_URL_FILE=""
TARGET_RUNS=20
DELAY_SECS=30
POLL_SECS=5
VERIFICATION_TIMEOUT_SECS=480
NORMAL_INTERVAL_SECS=600
NORMAL_RUN_SAFETY_SECS=60
VERIFICATION_OBJECT_WORKERS=4
VERIFICATION_CHILD_PATH=""

while [[ $# -gt 0 ]]; do
  case "$1" in
    --runs-root) RUNS_ROOT="${2:?missing value}"; shift 2 ;;
    --state-root) STATE_ROOT="${2:?missing value}"; shift 2 ;;
    --rpki-bin) RPKI_BIN="${2:?missing value}"; shift 2 ;;
    --work-root) WORK_ROOT="${2:?missing value}"; shift 2 ;;
    --webhook-url-file) WEBHOOK_URL_FILE="${2:?missing value}"; shift 2 ;;
    --target-runs) TARGET_RUNS="${2:?missing value}"; shift 2 ;;
    --delay-secs) DELAY_SECS="${2:?missing value}"; shift 2 ;;
    --poll-secs) POLL_SECS="${2:?missing value}"; shift 2 ;;
    --verification-timeout-secs) VERIFICATION_TIMEOUT_SECS="${2:?missing value}"; shift 2 ;;
    --normal-interval-secs) NORMAL_INTERVAL_SECS="${2:?missing value}"; shift 2 ;;
    --normal-run-safety-secs) NORMAL_RUN_SAFETY_SECS="${2:?missing value}"; shift 2 ;;
    --verification-object-workers) VERIFICATION_OBJECT_WORKERS="${2:?missing value}"; shift 2 ;;
    --verification-child-path) VERIFICATION_CHILD_PATH="${2:?missing value}"; shift 2 ;;
    -h|--help) usage; exit 0 ;;
    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
  esac
done

for value_name in RUNS_ROOT STATE_ROOT RPKI_BIN WORK_ROOT WEBHOOK_URL_FILE; do
  [[ -n "${!value_name}" ]] || { echo "$value_name is required" >&2; exit 2; }
done
for integer_name in TARGET_RUNS DELAY_SECS POLL_SECS VERIFICATION_TIMEOUT_SECS NORMAL_INTERVAL_SECS VERIFICATION_OBJECT_WORKERS; do
  [[ "${!integer_name}" =~ ^[0-9]+$ ]] && [[ "${!integer_name}" != 0 ]] || {
    echo "$integer_name must be a positive integer" >&2
    exit 2
  }
done
[[ "$NORMAL_RUN_SAFETY_SECS" =~ ^[0-9]+$ ]] || {
  echo "NORMAL_RUN_SAFETY_SECS must be a non-negative integer" >&2
  exit 2
}
(( NORMAL_RUN_SAFETY_SECS < NORMAL_INTERVAL_SECS )) || {
  echo "NORMAL_RUN_SAFETY_SECS must be smaller than NORMAL_INTERVAL_SECS" >&2
  exit 2
}
[[ -d "$RUNS_ROOT" ]] || { echo "runs root is not a directory: $RUNS_ROOT" >&2; exit 2; }
[[ -d "$STATE_ROOT" ]] || { echo "state root is not a directory: $STATE_ROOT" >&2; exit 2; }
[[ -x "$RPKI_BIN" ]] || { echo "rpki binary is not executable: $RPKI_BIN" >&2; exit 2; }
[[ -f "$WEBHOOK_URL_FILE" ]] || { echo "webhook URL file is absent: $WEBHOOK_URL_FILE" >&2; exit 2; }
for required_command in curl flock ionice jq nice python3 sha256sum timeout; do
  command -v "$required_command" >/dev/null || {
    echo "missing required command: $required_command" >&2
    exit 2
  }
done

mkdir -p "$WORK_ROOT/results" "$WORK_ROOT/scratch"
STATE_FILE="$WORK_ROOT/observer-state.json"
OBSERVER_LOCK="$WORK_ROOT/observer.lock"
RESULTS_DIR="$WORK_ROOT/results"
SCRATCH_ROOT="$WORK_ROOT/scratch"
WEBHOOK_URL="$(tr -d '\r\n' < "$WEBHOOK_URL_FILE")"
[[ "$WEBHOOK_URL" == https://open.feishu.cn/open-apis/bot/v2/hook/* ]] || {
  echo "webhook URL does not have the expected Feishu incoming-webhook form" >&2
  exit 2
}

now_utc() { date -u +%Y-%m-%dT%H:%M:%SZ; }
now_ms() { python3 -c 'import time; print(time.time_ns() // 1_000_000)'; }

remaining_verification_window_secs() {
  local completed_at="$1"
  python3 - "$completed_at" "$NORMAL_INTERVAL_SECS" "$NORMAL_RUN_SAFETY_SECS" <<'PY'
import datetime
import sys
import time

completed = datetime.datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00"))
interval = int(sys.argv[2])
safety = int(sys.argv[3])
print(int(completed.timestamp()) + interval - safety - int(time.time()))
PY
}

atomic_replace() {
  local destination="$1"
  local temporary="${destination}.tmp.$$"
  cat > "$temporary"
  mv "$temporary" "$destination"
}

terminal_run_ids() {
  local run_dir meta
  for run_dir in "$RUNS_ROOT"/run_[0-9]*; do
    [[ -d "$run_dir" ]] || continue
    meta="$run_dir/run-meta.json"
    [[ -f "$meta" ]] || continue
    jq -e '(.completed_at_rfc3339_utc? // .completedAt? // null) != null' "$meta" >/dev/null 2>&1 || continue
    basename "$run_dir"
  done | sort -t_ -k2,2n
}

run_number() {
  local run_id="$1"
  printf '%d\n' "$((10#${run_id#run_}))"
}

latest_terminal_run_id() {
  terminal_run_ids | tail -n 1
}

result_path() {
  printf '%s/%s.json\n' "$RESULTS_DIR" "$1"
}

read_json_or_empty() {
  local path="$1"
  if [[ -f "$path" ]] && jq -e . "$path" >/dev/null 2>&1; then
    jq -c . "$path"
  else
    printf '{}'
  fi
}

update_cumulative() {
  local observed normal_success passes mismatches errors skipped pending latest
  local -a result_files
  shopt -s nullglob
  result_files=("$RESULTS_DIR"/run_*.json)
  shopt -u nullglob
  observed="${#result_files[@]}"
  if (( observed == 0 )); then
    normal_success=0
    passes=0
    mismatches=0
    errors=0
    skipped=0
    pending=0
  else
    normal_success="$(jq -r 'select(.normalRun.status == "success") | 1' "${result_files[@]}" | wc -l)"
    passes="$(jq -r 'select(.verification.status == "pass") | 1' "${result_files[@]}" | wc -l)"
    mismatches="$(jq -r 'select(.verification.status == "mismatch") | 1' "${result_files[@]}" | wc -l)"
    errors="$(jq -r 'select(.verification.status == "error") | 1' "${result_files[@]}" | wc -l)"
    skipped="$(jq -r 'select(.verification.status | startswith("skipped_")) | 1' "${result_files[@]}" | wc -l)"
    pending="$(jq -r 'select(.notification.status != "sent") | 1' "${result_files[@]}" | wc -l)"
  fi
  latest="$(find "$RESULTS_DIR" -maxdepth 1 -type f -name 'run_*.json' -printf '%f\n' | sed 's/.json$//' | sort -t_ -k2,2n | tail -n 1)"
  jq -n \
    --arg updated_at "$(now_utc)" \
    --arg initial_run_id "$(jq -r '.initialLatestTerminalRunId' "$STATE_FILE")" \
    --arg latest_run_id "$latest" \
    --argjson target_runs "$TARGET_RUNS" \
    --argjson observed_runs "$observed" \
    --argjson normal_success_runs "$normal_success" \
    --argjson verification_passes "$passes" \
    --argjson verification_mismatches "$mismatches" \
    --argjson verification_errors "$errors" \
    --argjson verification_skipped "$skipped" \
    --argjson notification_pending "$pending" \
    '{schemaVersion: 1, updatedAt: $updated_at, initialLatestTerminalRunId: $initial_run_id,
      latestProcessedRunId: $latest_run_id, targetRuns: $target_runs, observedRuns: $observed_runs,
      normalSuccessRuns: $normal_success_runs, verificationPasses: $verification_passes,
      verificationMismatches: $verification_mismatches, verificationErrors: $verification_errors,
      verificationSkipped: $verification_skipped, notificationPending: $notification_pending}' \
    | atomic_replace "$RESULTS_DIR/cumulative.json"
}

notify_result() {
  local path="$1" run_id status observed target passes mismatches errors skipped analysis_available top_phase message payload response
  run_id="$(jq -r '.runId' "$path")"
  status="$(jq -r '.verification.status' "$path")"
  observed="$(jq -r '.observedRuns' "$RESULTS_DIR/cumulative.json")"
  target="$(jq -r '.targetRuns' "$RESULTS_DIR/cumulative.json")"
  passes="$(jq -r '.verificationPasses' "$RESULTS_DIR/cumulative.json")"
  mismatches="$(jq -r '.verificationMismatches' "$RESULTS_DIR/cumulative.json")"
  errors="$(jq -r '.verificationErrors' "$RESULTS_DIR/cumulative.json")"
  skipped="$(jq -r '.verificationSkipped' "$RESULTS_DIR/cumulative.json")"
  analysis_available="$(jq -r '.verification.analysis.available // false' "$path")"
  top_phase="$(jq -r '(.verification.analysis.phaseTotals // [])[0] | if . == null then "n/a" else "\(.name)=\(.totalMs)ms" end' "$path")"
  message="From codex: #137 远端231 verification-only 旁路复核\n${run_id}: ${status}\n"
  message+="CCR state digest match=$(jq -r 'if .verification.comparison | has("stateDigestMatch") then (.verification.comparison.stateDigestMatch | tostring) else "n/a" end' "$path"); fallback replay match=$(jq -r 'if .verification.comparison.fallbackReplaySelection | has("matches") then (.verification.comparison.fallbackReplaySelection.matches | tostring) else "n/a" end' "$path"); verificationProcessMs=$(jq -r '.verification.timing.verificationProcessMs // .verification.wallMs // "n/a"' "$path"); preValidationPreparationMs=$(jq -r '.verification.timing.preValidationPreparationMs // "n/a"' "$path"); lockWaitMs=$(jq -r '.verification.lockWaitMs // "n/a"' "$path")\n"
  message+="analysisAvailable=${analysis_available}; topPhase=${top_phase}\n"
  message+="累计 ${observed}/${target}: pass=${passes}, mismatch=${mismatches}, error=${errors}, skipped=${skipped}\n"
  message+="结果: ${path}"
  payload="$(jq -n --arg text "$message" '{msg_type: "text", content: {text: $text}}')"
  if response="$(curl --fail-with-body --silent --show-error --connect-timeout 10 --max-time 30 \
      -H 'Content-Type: application/json' --data "$payload" "$WEBHOOK_URL" 2>&1)"; then
    if jq -e '((.code // .StatusCode // 0) == 0)' <<< "$response" >/dev/null 2>&1; then
      jq --arg sent_at "$(now_utc)" --arg response "$response" \
        '.notification = {status: "sent", sentAt: $sent_at, response: $response}' "$path" \
        | atomic_replace "$path"
      return 0
    fi
    response="Feishu rejected payload: $response"
  fi
  jq --arg attempted_at "$(now_utc)" --arg error "$response" \
    '.notification = {status: "pending", lastAttemptAt: $attempted_at, lastError: $error}' "$path" \
    | atomic_replace "$path"
  return 1
}

deliver_pending_notifications() {
  local path
  for path in "$RESULTS_DIR"/run_*.json; do
    [[ -f "$path" ]] || continue
    [[ "$(jq -r '.notification.status // "pending"' "$path")" == "sent" ]] && continue
    update_cumulative
    notify_result "$path" || true
  done
  update_cumulative
}

write_result() {
  local run_id="$1" normal_json="$2" verification_json="$3" result
  result="$(result_path "$run_id")"
  jq -n \
    --arg recorded_at "$(now_utc)" \
    --arg run_id "$run_id" \
    --argjson normal_run "$normal_json" \
    --argjson verification "$verification_json" \
    '{schemaVersion: 1, recordedAt: $recorded_at, runId: $run_id, normalRun: $normal_run,
      verification: $verification, notification: {status: "pending"}}' \
    | atomic_replace "$result"
  update_cumulative
  notify_result "$result" || true
  update_cumulative
}

normal_run_summary() {
  local run_dir="$1"
  if [[ ! -f "$run_dir/run-meta.json" ]]; then
    printf '{"status":"invalid-meta"}'
    return
  fi
  jq -cn --slurpfile meta "$run_dir/run-meta.json" --slurpfile summary "$run_dir/run-summary.json" \
    '{status: ($meta[0].status // "unknown"), syncMode: ($meta[0].sync_mode // $meta[0].syncMode // "unknown"),
      runIndex: ($meta[0].run_index // $meta[0].runIndex // null),
      completedAt: ($meta[0].completed_at_rfc3339_utc // $meta[0].completedAt // null),
      wallMs: ($summary[0].wall_ms // $summary[0].wallMs // null),
      exitCode: ($summary[0].exit_code // $summary[0].exitCode // $meta[0].daemon_exit_code // null)}' 2>/dev/null \
    || printf '{"status":"invalid-meta"}'
}

process_terminal_run() {
  local run_id="$1" run_dir="$RUNS_ROOT/$run_id" normal_json status temporary_dir state_lock_fd lock_started_ms lock_acquired_ms
  local binding_run_id binary_sha source_sha run_started_ms run_finished_ms exit_code comparison_json comparison_state_digest_match fallback_replay_comparison_json fallback_replay_selection_match verification_meta_json stage_timing_json analysis_json analysis_summary_json error_tail max_rss_kb verification_status verification_json timeout_secs completed_at
  local -a verification_child_exec
  local observer_started_ms delay_finished_ms preparation_started_ms binding_started_ms binding_finished_ms binary_check_started_ms binary_check_finished_ms window_started_ms window_finished_ms scratch_cleanup_started_ms scratch_cleanup_finished_ms post_validation_started_ms post_validation_collection_finished_ms result_cleanup_started_ms result_cleanup_finished_ms
  normal_json="$(normal_run_summary "$run_dir")"
  status="$(jq -r '.status' <<< "$normal_json")"
  observer_started_ms="$(now_ms)"
  sleep "$DELAY_SECS"
  delay_finished_ms="$(now_ms)"
  if [[ "$status" != "success" ]]; then
    write_result "$run_id" "$normal_json" '{"status":"skipped_normal_run_not_success"}'
    return
  fi

  preparation_started_ms="$delay_finished_ms"
  mkdir -p "$STATE_ROOT/locks"
  exec {state_lock_fd}>"$STATE_ROOT/locks/state-execution.lock"
  lock_started_ms="$(now_ms)"
  if ! flock -n "$state_lock_fd"; then
    exec {state_lock_fd}>&-
    write_result "$run_id" "$normal_json" '{"status":"skipped_state_lock_busy"}'
    return
  fi
  lock_acquired_ms="$(now_ms)"
  binding_started_ms="$(now_ms)"
  binding_run_id="$(jq -r '.currentRunId // empty' "$STATE_ROOT/meta/current-state-binding.json" 2>/dev/null || true)"
  binding_finished_ms="$(now_ms)"
  if [[ "$binding_run_id" != "$run_id" ]]; then
    flock -u "$state_lock_fd"
    exec {state_lock_fd}>&-
    verification_json="$(jq -n --arg bound_run "$binding_run_id" --argjson lock_wait_ms "$((lock_acquired_ms - lock_started_ms))" \
      '{status: "skipped_source_no_longer_current", boundRunId: $bound_run, lockWaitMs: $lock_wait_ms}')"
    write_result "$run_id" "$normal_json" "$verification_json"
    return
  fi
  binary_check_started_ms="$(now_ms)"
  source_sha="$(jq -r '.binarySha256 // empty' "$run_dir/validation-contract.json" 2>/dev/null || true)"
  binary_sha="$(sha256sum "$RPKI_BIN" | awk '{print $1}')"
  binary_check_finished_ms="$(now_ms)"
  if [[ -z "$source_sha" || "$source_sha" != "$binary_sha" ]]; then
    flock -u "$state_lock_fd"
    exec {state_lock_fd}>&-
    verification_json="$(jq -n --arg expected "$source_sha" --arg actual "$binary_sha" --argjson lock_wait_ms "$((lock_acquired_ms - lock_started_ms))" \
      '{status: "error_binary_sha256_mismatch", expectedBinarySha256: $expected, actualBinarySha256: $actual, lockWaitMs: $lock_wait_ms}')"
    write_result "$run_id" "$normal_json" "$verification_json"
    return
  fi

  window_started_ms="$(now_ms)"
  completed_at="$(jq -r '.completedAt // empty' <<< "$normal_json")"
  timeout_secs="$(remaining_verification_window_secs "$completed_at")"
  window_finished_ms="$(now_ms)"
  if (( timeout_secs <= 0 )); then
    flock -u "$state_lock_fd"
    exec {state_lock_fd}>&-
    verification_json="$(jq -n --arg completed_at "$completed_at" --argjson lock_wait_ms "$((lock_acquired_ms - lock_started_ms))" \
      '{status: "skipped_insufficient_interval", sourceCompletedAt: $completed_at, lockWaitMs: $lock_wait_ms}')"
    write_result "$run_id" "$normal_json" "$verification_json"
    return
  fi
  if (( timeout_secs > VERIFICATION_TIMEOUT_SECS )); then
    timeout_secs="$VERIFICATION_TIMEOUT_SECS"
  fi

  temporary_dir="$SCRATCH_ROOT/$run_id"
  scratch_cleanup_started_ms="$(now_ms)"
  rm -rf "$temporary_dir"
  scratch_cleanup_finished_ms="$(now_ms)"
  run_started_ms="$(now_ms)"
  verification_child_exec=()
  if [[ -n "$VERIFICATION_CHILD_PATH" ]]; then
    verification_child_exec=(/usr/bin/env "PATH=$VERIFICATION_CHILD_PATH")
  fi
  set +e
  RPKI_STATE_EXECUTION_LOCK_HELD=1 /usr/bin/time -v -o "$temporary_dir.process-time" -- \
    /usr/bin/timeout --foreground --signal=TERM --kill-after=30s "$timeout_secs" \
      /usr/bin/ionice -c2 -n0 "${verification_child_exec[@]}" "$RPKI_BIN" \
        --verification-only --verification-source-run "$run_dir" \
        --verification-source-state "$STATE_ROOT" --verification-out "$temporary_dir" \
        --analyze --analysis-out "$temporary_dir/analyze" \
        --parallel-phase2-object-workers "$VERIFICATION_OBJECT_WORKERS" \
        >"$temporary_dir.stdout" 2>"$temporary_dir.stderr"
  exit_code=$?
  set -e
  run_finished_ms="$(now_ms)"
  post_validation_started_ms="$run_finished_ms"
  flock -u "$state_lock_fd"
  exec {state_lock_fd}>&-

  comparison_json="$(read_json_or_empty "$temporary_dir/compare/ccr-state-digest.json")"
  fallback_replay_comparison_json="$(read_json_or_empty "$temporary_dir/compare/fallback-replay-selection.json")"
  verification_meta_json="$(read_json_or_empty "$temporary_dir/verification-meta.json")"
  stage_timing_json="$(read_json_or_empty "$temporary_dir/stage-timing.json")"
  analysis_json="$(read_json_or_empty "$temporary_dir/analyze/timing.json")"
  analysis_summary_json="$(jq -cn --argjson stage "$stage_timing_json" --argjson analysis "$analysis_json" '
    {enabled: true,
     available: (($analysis | type) == "object" and ($analysis | length) > 0),
     stageTiming: {
       totalMs: ($stage.total_ms // null),
       validationMs: ($stage.validation_ms // null),
       repoSyncMsTotal: ($stage.repo_sync_ms_total // null),
       publicationPointRepoSyncMsTotal: ($stage.publication_point_repo_sync_ms_total // null),
       rrdpDownloadMsTotal: ($stage.rrdp_download_ms_total // null),
       rsyncDownloadMsTotal: ($stage.rsync_download_ms_total // null),
       reportBuildMs: ($stage.report_build_ms // null),
       reportWriteMs: ($stage.report_write_ms // null),
       ccrBuildMs: ($stage.ccr_build_ms // null),
       ccrWriteMs: ($stage.ccr_write_ms // null),
       cirTotalMs: ($stage.cir_total_ms // null)
     },
     phaseTotals: (($analysis.phases // {})
       | to_entries
       | map({name: .key, count: (.value.count // 0), totalMs: (((.value.total_nanos // 0) / 1000000) | floor)})
       | sort_by(-.totalMs)
       | .[0:20]),
     topPublicationPoints: (($analysis.top_publication_points // [])[0:10]),
     topPublicationPointSteps: (($analysis.top_publication_point_steps // [])[0:10]),
     topRrdpRepos: (($analysis.top_rrdp_repos // [])[0:10]),
     topRrdpRepoSteps: (($analysis.top_rrdp_repo_steps // [])[0:10])}')"
  error_tail="$(tail -n 20 "$temporary_dir.stderr" 2>/dev/null || true)"
  max_rss_kb="$(awk -F: '/Maximum resident set size/{gsub(/^[[:space:]]+/, "", $2); print $2}' "$temporary_dir.process-time" 2>/dev/null || true)"
  comparison_state_digest_match="$(jq -r 'if has("stateDigestMatch") then (.stateDigestMatch | tostring) else "missing" end' <<< "$comparison_json")"
  fallback_replay_selection_match="$(jq -r 'if has("matches") then (.matches | tostring) else "missing" end' <<< "$fallback_replay_comparison_json")"
  if [[ "$comparison_state_digest_match" == "true" && "$fallback_replay_selection_match" == "true" && "$exit_code" -eq 0 ]]; then
    verification_status="pass"
  elif [[ "$comparison_state_digest_match" == "false" || "$fallback_replay_selection_match" == "false" ]]; then
    verification_status="mismatch"
  else
    verification_status="error"
  fi
  post_validation_collection_finished_ms="$(now_ms)"
  result_cleanup_started_ms="$(now_ms)"
  rm -rf "$temporary_dir" "$temporary_dir.stdout" "$temporary_dir.stderr" "$temporary_dir.process-time"
  result_cleanup_finished_ms="$(now_ms)"
  verification_json="$(jq -n \
    --arg status "$verification_status" \
    --arg verification_child_path "$VERIFICATION_CHILD_PATH" \
    --argjson exit_code "$exit_code" \
    --argjson wall_ms "$((run_finished_ms - run_started_ms))" \
    --argjson lock_wait_ms "$((lock_acquired_ms - lock_started_ms))" \
    --argjson timeout_limit_secs "$timeout_secs" \
    --argjson object_workers "$VERIFICATION_OBJECT_WORKERS" \
    --arg max_rss_kb "${max_rss_kb:-}" \
    --arg error_tail "$error_tail" \
    --argjson comparison "$comparison_json" \
    --argjson fallback_replay_comparison "$fallback_replay_comparison_json" \
    --argjson meta "$verification_meta_json" \
    --argjson analysis "$analysis_summary_json" \
    --argjson post_run_delay_ms "$((delay_finished_ms - observer_started_ms))" \
    --argjson observer_to_verification_start_ms "$((run_started_ms - observer_started_ms))" \
    --argjson pre_validation_preparation_ms "$((run_started_ms - preparation_started_ms))" \
    --argjson binding_check_ms "$((binding_finished_ms - binding_started_ms))" \
    --argjson binary_contract_check_ms "$((binary_check_finished_ms - binary_check_started_ms))" \
    --argjson interval_window_calculation_ms "$((window_finished_ms - window_started_ms))" \
    --argjson scratch_preparation_cleanup_ms "$((scratch_cleanup_finished_ms - scratch_cleanup_started_ms))" \
    --argjson verification_process_ms "$((run_finished_ms - run_started_ms))" \
    --argjson post_validation_collection_ms "$((post_validation_collection_finished_ms - post_validation_started_ms))" \
    --argjson result_cleanup_ms "$((result_cleanup_finished_ms - result_cleanup_started_ms))" \
    '{status: $status, exitCode: $exit_code, wallMs: $wall_ms, lockWaitMs: $lock_wait_ms,
      timeoutLimitSecs: $timeout_limit_secs, objectWorkers: $object_workers,
      verificationChildPath: (if $verification_child_path == "" then null else $verification_child_path end),
      timing: {postRunDelayMs: $post_run_delay_ms,
               observerToVerificationStartMs: $observer_to_verification_start_ms,
               preValidationPreparationMs: $pre_validation_preparation_ms,
               preparationBreakdownMs: {stateLockWaitMs: $lock_wait_ms,
                                        bindingCheckMs: $binding_check_ms,
                                        binaryContractCheckMs: $binary_contract_check_ms,
                                        intervalWindowCalculationMs: $interval_window_calculation_ms,
                                        scratchPreparationCleanupMs: $scratch_preparation_cleanup_ms},
               verificationProcessMs: $verification_process_ms,
               postValidationCollectionMs: $post_validation_collection_ms,
               resultCleanupMs: $result_cleanup_ms},
      maxRssKb: (if $max_rss_kb == "" then null else ($max_rss_kb | tonumber) end),
      comparison: {stateDigestMatch: ($comparison | if has("stateDigestMatch") then .stateDigestMatch else null end),
                   mismatchedStates: ($comparison.mismatchedStates // []),
                   states: (($comparison.states // []) | map({name, sourcePresent, verificationPresent,
                                                               sourceHashHex, verificationHashHex, matches})),
                   fallbackReplaySelection: {
                     matches: ($fallback_replay_comparison | if has("matches") then .matches else null end),
                     expectedCount: (($fallback_replay_comparison.expected // []) | length),
                     actualCount: (($fallback_replay_comparison.actual // []) | length),
                     missing: ($fallback_replay_comparison.missing // []),
                     unexpected: ($fallback_replay_comparison.unexpected // [])
                   }},
      verificationMeta: {status: ($meta.status // null), wallMs: ($meta.wallMs // null),
                         repositoryObjectsChecked: ($meta.repositoryObjectsChecked // null),
                         repositoryBytesChecked: ($meta.repositoryBytesChecked // null)},
      analysis: $analysis,
      errorTail: $error_tail}')"
  write_result "$run_id" "$normal_json" "$verification_json"
}

exec {observer_lock_fd}>"$OBSERVER_LOCK"
if ! flock -n "$observer_lock_fd"; then
  echo "another observer already owns $OBSERVER_LOCK" >&2
  exit 3
fi

find "$SCRATCH_ROOT" -mindepth 1 -maxdepth 1 -exec rm -rf {} +

if [[ ! -f "$STATE_FILE" ]]; then
  initial_run_id="$(latest_terminal_run_id)"
  [[ -n "$initial_run_id" ]] || { echo "no terminal normal run found under $RUNS_ROOT" >&2; exit 2; }
  jq -n \
    --arg started_at "$(now_utc)" \
    --arg initial_run_id "$initial_run_id" \
    --arg runs_root "$RUNS_ROOT" \
    --arg state_root "$STATE_ROOT" \
    --arg rpki_sha256 "$(sha256sum "$RPKI_BIN" | awk '{print $1}')" \
    --argjson target_runs "$TARGET_RUNS" \
    --argjson delay_secs "$DELAY_SECS" \
    --argjson verification_timeout_secs "$VERIFICATION_TIMEOUT_SECS" \
    --argjson normal_interval_secs "$NORMAL_INTERVAL_SECS" \
    --argjson normal_run_safety_secs "$NORMAL_RUN_SAFETY_SECS" \
    --argjson verification_object_workers "$VERIFICATION_OBJECT_WORKERS" \
    --arg verification_child_path "$VERIFICATION_CHILD_PATH" \
    '{schemaVersion: 1, startedAt: $started_at, initialLatestTerminalRunId: $initial_run_id,
      runsRoot: $runs_root, stateRoot: $state_root, verifierBinarySha256: $rpki_sha256,
      targetRuns: $target_runs, delaySecs: $delay_secs, verificationTimeoutSecs: $verification_timeout_secs,
      normalIntervalSecs: $normal_interval_secs, normalRunSafetySecs: $normal_run_safety_secs,
      verificationObjectWorkers: $verification_object_workers,
      verificationChildPath: (if $verification_child_path == "" then null else $verification_child_path end),
      verificationAnalyze: true}' \
    | atomic_replace "$STATE_FILE"
  update_cumulative
fi

# Preserve the original baseline run while recording the live configuration after
# an observer restart. This is particularly important for the normal-run safety
# window, whose length depends on the soak interval.
jq --arg updated_at "$(now_utc)" \
  --argjson target_runs "$TARGET_RUNS" \
  --argjson delay_secs "$DELAY_SECS" \
  --argjson verification_timeout_secs "$VERIFICATION_TIMEOUT_SECS" \
  --argjson normal_interval_secs "$NORMAL_INTERVAL_SECS" \
  --argjson normal_run_safety_secs "$NORMAL_RUN_SAFETY_SECS" \
  --argjson verification_object_workers "$VERIFICATION_OBJECT_WORKERS" \
  --arg verification_child_path "$VERIFICATION_CHILD_PATH" \
  '.updatedAt = $updated_at
   | .targetRuns = $target_runs
   | .delaySecs = $delay_secs
   | .verificationTimeoutSecs = $verification_timeout_secs
   | .normalIntervalSecs = $normal_interval_secs
   | .normalRunSafetySecs = $normal_run_safety_secs
   | .verificationObjectWorkers = $verification_object_workers
   | .verificationChildPath = (if $verification_child_path == "" then null else $verification_child_path end)
   | .verificationAnalyze = true' "$STATE_FILE" \
  | atomic_replace "$STATE_FILE"

initial_run_number="$(run_number "$(jq -r '.initialLatestTerminalRunId' "$STATE_FILE")")"
while true; do
  deliver_pending_notifications
  observed_count="$(find "$RESULTS_DIR" -maxdepth 1 -type f -name 'run_*.json' -printf '%f\n' | wc -l)"
  pending_count="$(jq -r '.notificationPending' "$RESULTS_DIR/cumulative.json")"
  if (( observed_count >= TARGET_RUNS && pending_count == 0 )); then
    break
  fi
  if (( observed_count < TARGET_RUNS )); then
    while IFS= read -r run_id; do
      [[ -n "$run_id" ]] || continue
      (( $(run_number "$run_id") > initial_run_number )) || continue
      [[ -f "$(result_path "$run_id")" ]] && continue
      process_terminal_run "$run_id"
      observed_count="$(find "$RESULTS_DIR" -maxdepth 1 -type f -name 'run_*.json' -printf '%f\n' | wc -l)"
      (( observed_count < TARGET_RUNS )) || break
    done < <(terminal_run_ids)
  fi
  sleep "$POLL_SECS"
done

update_cumulative
echo "completed $(jq -r '.observedRuns' "$RESULTS_DIR/cumulative.json") observed runs; results: $RESULTS_DIR/cumulative.json"