20260526 增加持续soak监控与本地回放工具

2026-05-26 18:02:40 +08:00
parent cda7fdb135
commit 7e1c24fcc3
30 changed files with 4541 additions and 12 deletions
--- a/scripts/local_repo_replay/build_local_repo_replay_package.sh
+++ b/scripts/local_repo_replay/build_local_repo_replay_package.sh
@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  build_local_repo_replay_package.sh --out <path> [--tar-gz]
+
+Build a standalone local repository tree replay package for Routinator and
+rpki-client. The package does not include repository data and does not include
+materialize tooling.
+EOF
+}
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+SRC_DIR="$ROOT_DIR/scripts/local_repo_replay"
+OUT=""
+TAR_GZ=0
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --out) OUT="$2"; shift 2 ;;
+    --tar-gz) TAR_GZ=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$OUT" ]] || { usage >&2; exit 2; }
+
+if [[ "$TAR_GZ" -eq 1 ]]; then
+  PACKAGE_DIR="$(mktemp -d)"
+  TARGET_DIR="$PACKAGE_DIR/local-repo-replay-package"
+else
+  TARGET_DIR="$OUT"
+  rm -rf "$TARGET_DIR"
+fi
+
+mkdir -p "$TARGET_DIR/scripts" "$TARGET_DIR/docs" "$TARGET_DIR/examples"
+
+install -m 0755 "$SRC_DIR/run_routinator_from_local_tree.sh" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/run_rpki_client_from_local_tree.sh" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/run_dual_local_tree_replay.sh" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/prepare_tals.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/normalize_rp_outputs.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/compare_normalized_sets.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$SRC_DIR/summarize_replay.py" "$TARGET_DIR/scripts/"
+install -m 0755 "$ROOT_DIR/scripts/cir/cir-rsync-wrapper" "$TARGET_DIR/scripts/"
+install -m 0755 "$ROOT_DIR/scripts/cir/cir-local-link-sync.py" "$TARGET_DIR/scripts/"
+install -m 0644 "$SRC_DIR/templates/README.md" "$TARGET_DIR/README.md"
+install -m 0644 "$SRC_DIR/templates/docs/input_tree_requirements.md" "$TARGET_DIR/docs/"
+install -m 0644 "$SRC_DIR/templates/docs/offline_replay_limits.md" "$TARGET_DIR/docs/"
+install -m 0644 "$SRC_DIR/templates/docs/output_files.md" "$TARGET_DIR/docs/"
+install -m 0755 "$SRC_DIR/templates/examples/routinator_example.sh" "$TARGET_DIR/examples/"
+install -m 0755 "$SRC_DIR/templates/examples/rpki_client_example.sh" "$TARGET_DIR/examples/"
+install -m 0755 "$SRC_DIR/templates/examples/dual_compare_example.sh" "$TARGET_DIR/examples/"
+cat > "$TARGET_DIR/env.example" <<'EOF'
+# 本地目录树 replay 示例配置。目录树由使用者提前准备，不包含在 package 中。
+TAL_DIR=/data/replay/tals
+MIRROR_ROOT=/data/replay/mirror
+ROUTINATOR_BIN=/opt/routinator/target/release/routinator
+RPKI_CLIENT_BIN=/opt/rpki-client/src/rpki-client
+RPKI_CLIENT_CACHE_DIR=/data/replay/work/rpki-client-cache
+VALIDATION_TIME=2026-05-23T00:00:00Z
+EOF
+
+if grep -R -E 'cir_materialize|repo-bytes\\.db|\\.cir' "$TARGET_DIR/scripts" >/dev/null; then
+  echo "package contains forbidden materialize/repo-bytes implementation references" >&2
+  exit 1
+fi
+
+if [[ "$TAR_GZ" -eq 1 ]]; then
+  mkdir -p "$(dirname "$OUT")"
+  tar -C "$PACKAGE_DIR" -czf "$OUT" local-repo-replay-package
+  rm -rf "$PACKAGE_DIR"
+  echo "$OUT"
+else
+  echo "$TARGET_DIR"
+fi
--- a/scripts/local_repo_replay/compare_normalized_sets.py
+++ b/scripts/local_repo_replay/compare_normalized_sets.py
@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def read_set(path: Path) -> set[str]:
+    if not path.is_file():
+        return set()
+    return {line.strip() for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip()}
+
+
+def compare(left: set[str], right: set[str]) -> dict[str, object]:
+    union = left | right
+    inter = left & right
+    return {
+        "left": len(left),
+        "right": len(right),
+        "intersection": len(inter),
+        "onlyLeft": len(left - right),
+        "onlyRight": len(right - left),
+        "jaccard": round(len(inter) / len(union), 8) if union else 1.0,
+        "onlyLeftSamples": sorted(left - right)[:20],
+        "onlyRightSamples": sorted(right - left)[:20],
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--left", type=Path, required=True)
+    parser.add_argument("--right", type=Path, required=True)
+    parser.add_argument("--left-name", default="left")
+    parser.add_argument("--right-name", default="right")
+    parser.add_argument("--out", type=Path, required=True)
+    args = parser.parse_args()
+    result = {
+        "leftName": args.left_name,
+        "rightName": args.right_name,
+        "vrps": compare(read_set(args.left / "vrps.normalized.txt"), read_set(args.right / "vrps.normalized.txt")),
+        "vaps": compare(read_set(args.left / "vaps.normalized.txt"), read_set(args.right / "vaps.normalized.txt")),
+    }
+    args.out.parent.mkdir(parents=True, exist_ok=True)
+    args.out.write_text(json.dumps(result, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print(args.out)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/scripts/local_repo_replay/normalize_rp_outputs.py
+++ b/scripts/local_repo_replay/normalize_rp_outputs.py
@@ -0,0 +1,123 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+from pathlib import Path
+from typing import Any
+
+
+def normalize_asn(value: Any) -> str:
+    text = str(value).strip().upper()
+    if text.startswith("AS"):
+        text = text[2:]
+    return f"AS{int(text)}"
+
+
+def write_set(path: Path, rows: set[str]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text("\n".join(sorted(rows)) + ("\n" if rows else ""), encoding="utf-8")
+
+
+def load_json(path: Path) -> Any:
+    return json.loads(path.read_text(encoding="utf-8"))
+
+
+def normalize_vrps_from_json(path: Path) -> set[str]:
+    data = load_json(path)
+    rows: set[str] = set()
+    objects: list[dict[str, Any]] = []
+    if isinstance(data, dict):
+        for key in ("roas", "routeOrigins", "valid_roas"):
+            value = data.get(key)
+            if isinstance(value, list):
+                objects.extend(item for item in value if isinstance(item, dict))
+    elif isinstance(data, list):
+        objects = [item for item in data if isinstance(item, dict)]
+    for item in objects:
+        asn = item.get("asn") or item.get("asID") or item.get("origin")
+        prefix = item.get("prefix")
+        max_length = item.get("maxLength") or item.get("max_length") or item.get("maxlen") or item.get("maxLengthPrefix")
+        if asn is None or prefix is None or max_length is None:
+            continue
+        rows.add(f"{normalize_asn(asn)}|{prefix}|{int(max_length)}")
+    return rows
+
+
+def normalize_vaps_from_json(path: Path) -> set[str]:
+    data = load_json(path)
+    rows: set[str] = set()
+    objects: list[dict[str, Any]] = []
+    if isinstance(data, dict):
+        for key in ("aspas", "aspaAssertions", "vaps"):
+            value = data.get(key)
+            if isinstance(value, list):
+                objects.extend(item for item in value if isinstance(item, dict))
+    elif isinstance(data, list):
+        objects = [item for item in data if isinstance(item, dict)]
+    for item in objects:
+        customer = (
+            item.get("customer")
+            or item.get("customer_asid")
+            or item.get("customerASID")
+            or item.get("customerAsid")
+            or item.get("customerAsn")
+        )
+        providers = item.get("providers") or item.get("provider_asns") or item.get("providerASNs") or []
+        if customer is None:
+            continue
+        provider_asns = [normalize_asn(provider) for provider in providers]
+        rows.add(f"{normalize_asn(customer)}|{','.join(sorted(set(provider_asns), key=lambda value: int(value[2:])))}")
+    return rows
+
+
+def normalize_vrps_from_csv(path: Path) -> set[str]:
+    rows: set[str] = set()
+    with path.open(newline="", encoding="utf-8", errors="replace") as handle:
+        for row in csv.DictReader(handle):
+            asn = row.get("ASN") or row.get("asn") or row.get("AS")
+            prefix = row.get("IP Prefix") or row.get("prefix") or row.get("Prefix")
+            max_length = row.get("Max Length") or row.get("maxLength") or row.get("max_length")
+            if asn and prefix and max_length:
+                rows.add(f"{normalize_asn(asn)}|{prefix}|{int(max_length)}")
+    return rows
+
+
+def normalize_routinator(input_path: Path) -> tuple[set[str], set[str]]:
+    return normalize_vrps_from_json(input_path), normalize_vaps_from_json(input_path)
+
+
+def normalize_rpki_client(input_path: Path) -> tuple[set[str], set[str]]:
+    json_path = input_path / "json" if input_path.is_dir() else input_path
+    csv_path = input_path / "csv" if input_path.is_dir() else input_path
+    vrps: set[str] = set()
+    vaps: set[str] = set()
+    if json_path.is_file():
+        vrps |= normalize_vrps_from_json(json_path)
+        vaps |= normalize_vaps_from_json(json_path)
+    if csv_path.is_file():
+        vrps |= normalize_vrps_from_csv(csv_path)
+    return vrps, vaps
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--kind", choices=["routinator", "rpki-client"], required=True)
+    parser.add_argument("--input", type=Path, required=True)
+    parser.add_argument("--vrps-out", type=Path, required=True)
+    parser.add_argument("--vaps-out", type=Path, required=True)
+    args = parser.parse_args()
+
+    if args.kind == "routinator":
+        vrps, vaps = normalize_routinator(args.input)
+    else:
+        vrps, vaps = normalize_rpki_client(args.input)
+    write_set(args.vrps_out, vrps)
+    write_set(args.vaps_out, vaps)
+    print(json.dumps({"vrps": len(vrps), "vaps": len(vaps)}, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/scripts/local_repo_replay/prepare_tals.py
+++ b/scripts/local_repo_replay/prepare_tals.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+
+def split_tal(text: str) -> tuple[list[str], list[str]]:
+    lines = text.splitlines()
+    for index, line in enumerate(lines):
+        if line.strip() == "":
+            return lines[:index], lines[index + 1 :]
+    return lines, []
+
+
+def rsync_only_tal(source: Path) -> tuple[str, dict[str, object]]:
+    text = source.read_text(encoding="utf-8", errors="replace")
+    uri_lines, key_lines = split_tal(text)
+    uris = [
+        line.strip()
+        for line in uri_lines
+        if line.strip() and not line.lstrip().startswith("#")
+    ]
+    rsync_uris = [
+        uri for uri in uris
+        if uri.lower().startswith("rsync://")
+    ]
+    if not rsync_uris:
+        return text if text.endswith("\n") else text + "\n", {
+            "file": source.name,
+            "mode": "unchanged_no_rsync_uri",
+            "uris": len(uris),
+            "rsyncUris": 0,
+        }
+    out = "\n".join(rsync_uris) + "\n\n" + "\n".join(key_lines).strip() + "\n"
+    return out, {
+        "file": source.name,
+        "mode": "rsync_only",
+        "uris": len(uris),
+        "rsyncUris": len(rsync_uris),
+    }
+
+
+def collect_tals(tal_dir: Path | None, tals: list[Path]) -> list[Path]:
+    paths: list[Path] = []
+    if tal_dir is not None:
+        paths.extend(sorted(tal_dir.glob("*.tal")))
+    paths.extend(tals)
+    unique: dict[str, Path] = {}
+    for path in paths:
+        unique[path.name] = path
+    return [unique[name] for name in sorted(unique)]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Prepare TALs for local rsync-tree replay.")
+    parser.add_argument("--tal-dir", type=Path)
+    parser.add_argument("--tal", type=Path, action="append", default=[])
+    parser.add_argument("--out-dir", type=Path, required=True)
+    parser.add_argument("--summary", type=Path)
+    args = parser.parse_args()
+
+    sources = collect_tals(args.tal_dir, args.tal)
+    if not sources:
+        raise SystemExit("no TAL files provided")
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    rows = []
+    for source in sources:
+        if not source.is_file():
+            raise SystemExit(f"TAL file not found: {source}")
+        content, row = rsync_only_tal(source)
+        (args.out_dir / source.name).write_text(content, encoding="utf-8")
+        rows.append(row)
+
+    if args.summary is not None:
+        args.summary.parent.mkdir(parents=True, exist_ok=True)
+        args.summary.write_text(json.dumps(rows, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/scripts/local_repo_replay/run_dual_local_tree_replay.sh
+++ b/scripts/local_repo_replay/run_dual_local_tree_replay.sh
@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  run_dual_local_tree_replay.sh \
+    --routinator-bin <path> --routinator-mirror-root <dir> \
+    --rpki-client-bin <path> --rpki-client-mirror-root <dir> \
+    --tal-dir <dir> --out-dir <dir> [--validation-time <RFC3339>]
+EOF
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROUTINATOR_BIN=""
+ROUTINATOR_MIRROR_ROOT=""
+RPKI_CLIENT_BIN=""
+RPKI_CLIENT_MIRROR_ROOT=""
+RPKI_CLIENT_CACHE_DIR=""
+TAL_DIR=""
+OUT_DIR=""
+VALIDATION_TIME=""
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --routinator-mirror-root) ROUTINATOR_MIRROR_ROOT="$2"; shift 2 ;;
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --rpki-client-mirror-root) RPKI_CLIENT_MIRROR_ROOT="$2"; shift 2 ;;
+    --rpki-client-cache-dir) RPKI_CLIENT_CACHE_DIR="$2"; shift 2 ;;
+    --tal-dir) TAL_DIR="$2"; shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --validation-time) VALIDATION_TIME="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+if [[ -z "$RPKI_CLIENT_MIRROR_ROOT" ]]; then
+  RPKI_CLIENT_MIRROR_ROOT="$ROUTINATOR_MIRROR_ROOT"
+fi
+[[ -n "$ROUTINATOR_BIN" && -n "$ROUTINATOR_MIRROR_ROOT" && -n "$RPKI_CLIENT_BIN" && -n "$RPKI_CLIENT_MIRROR_ROOT" && -n "$TAL_DIR" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+
+mkdir -p "$OUT_DIR"
+TIME_ARGS=()
+if [[ -n "$VALIDATION_TIME" ]]; then
+  TIME_ARGS=(--validation-time "$VALIDATION_TIME")
+fi
+CACHE_ARGS=()
+if [[ -n "$RPKI_CLIENT_CACHE_DIR" ]]; then
+  CACHE_ARGS=(--cache-dir "$RPKI_CLIENT_CACHE_DIR")
+fi
+
+"$SCRIPT_DIR/run_routinator_from_local_tree.sh" \
+  --routinator-bin "$ROUTINATOR_BIN" \
+  --mirror-root "$ROUTINATOR_MIRROR_ROOT" \
+  --tal-dir "$TAL_DIR" \
+  --out-dir "$OUT_DIR/routinator" \
+  --enable-aspa \
+  "${TIME_ARGS[@]}"
+
+"$SCRIPT_DIR/run_rpki_client_from_local_tree.sh" \
+  --rpki-client-bin "$RPKI_CLIENT_BIN" \
+  --mirror-root "$RPKI_CLIENT_MIRROR_ROOT" \
+  --tal-dir "$TAL_DIR" \
+  --out-dir "$OUT_DIR/rpki-client" \
+  "${CACHE_ARGS[@]}" \
+  "${TIME_ARGS[@]}"
+
+python3 "$SCRIPT_DIR/compare_normalized_sets.py" \
+  --left "$OUT_DIR/routinator" \
+  --right "$OUT_DIR/rpki-client" \
+  --left-name routinator \
+  --right-name rpki-client \
+  --out "$OUT_DIR/compare-summary.json"
+echo "done: $OUT_DIR"
--- a/scripts/local_repo_replay/run_routinator_from_local_tree.sh
+++ b/scripts/local_repo_replay/run_routinator_from_local_tree.sh
@@ -0,0 +1,131 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  run_routinator_from_local_tree.sh \
+    --routinator-bin <path> \
+    --mirror-root <local-rsync-mirror-root> \
+    --tal-dir <dir> | --tal <file>... \
+    --out-dir <path> \
+    [--validation-time <RFC3339>] \
+    [--real-rsync-bin <path>] \
+    [--enable-aspa]
+
+The input mirror is prepared by the caller. This script does not generate it.
+Pass --validation-time only if FAKETIME_LIB points to a working libfaketime
+library; otherwise Routinator will validate at wall-clock time.
+Example:
+  export FAKETIME_LIB=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1
+EOF
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROUTINATOR_BIN=""
+MIRROR_ROOT=""
+OUT_DIR=""
+VALIDATION_TIME=""
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+ENABLE_ASPA=0
+TAL_DIR=""
+TALS=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --routinator-bin) ROUTINATOR_BIN="$2"; shift 2 ;;
+    --mirror-root) MIRROR_ROOT="$2"; shift 2 ;;
+    --tal-dir) TAL_DIR="$2"; shift 2 ;;
+    --tal) TALS+=("$2"); shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --validation-time) VALIDATION_TIME="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    --enable-aspa) ENABLE_ASPA=1; shift ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$ROUTINATOR_BIN" && -n "$MIRROR_ROOT" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+[[ -x "$ROUTINATOR_BIN" ]] || { echo "routinator binary not executable: $ROUTINATOR_BIN" >&2; exit 2; }
+[[ -d "$MIRROR_ROOT" ]] || { echo "mirror root not found: $MIRROR_ROOT" >&2; exit 2; }
+if [[ -z "$TAL_DIR" && "${#TALS[@]}" -eq 0 ]]; then
+  echo "provide --tal-dir or at least one --tal" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+WORK_DIR="$OUT_DIR/work"
+REPO_DIR="$WORK_DIR/repository"
+EXTRA_TALS="$WORK_DIR/tals"
+CONFIG_FILE="$WORK_DIR/routinator.conf"
+rm -rf "$WORK_DIR"
+mkdir -p "$REPO_DIR" "$EXTRA_TALS"
+
+PREPARE_TAL_ARGS=(--out-dir "$EXTRA_TALS" --summary "$OUT_DIR/prepared-tals.json")
+if [[ -n "$TAL_DIR" ]]; then
+  PREPARE_TAL_ARGS+=(--tal-dir "$TAL_DIR")
+fi
+for tal in "${TALS[@]}"; do
+  PREPARE_TAL_ARGS+=(--tal "$tal")
+done
+python3 "$SCRIPT_DIR/prepare_tals.py" "${PREPARE_TAL_ARGS[@]}"
+
+cat > "$CONFIG_FILE" <<EOF
+repository-dir = "$REPO_DIR"
+no-rir-tals = true
+extra-tals-dir = "$EXTRA_TALS"
+disable-rrdp = true
+rrdp-fallback = "never"
+rsync-command = "$SCRIPT_DIR/cir-rsync-wrapper"
+EOF
+
+export CIR_MIRROR_ROOT="$(cd "$MIRROR_ROOT" && pwd)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+export LOCAL_REPO_REPLAY_VALIDATION_TIME="$VALIDATION_TIME"
+export LOCAL_REPO_REPLAY_OUT_DIR="$OUT_DIR"
+
+ARGS=(
+  "$ROUTINATOR_BIN"
+  --config "$CONFIG_FILE"
+  --repository-dir "$REPO_DIR"
+  --disable-rrdp
+  --rrdp-fallback never
+  --rsync-command "$SCRIPT_DIR/cir-rsync-wrapper"
+  --no-rir-tals
+  --extra-tals-dir "$EXTRA_TALS"
+)
+if [[ "$ENABLE_ASPA" -eq 1 ]]; then
+  echo 'enable-aspa = true' >> "$CONFIG_FILE"
+  ARGS+=(--enable-aspa)
+fi
+if [[ -n "$VALIDATION_TIME" && -z "${FAKETIME_LIB:-}" ]]; then
+  echo "warning: --validation-time is ignored for Routinator because FAKETIME_LIB is not set" >&2
+fi
+VRP_ARGS=(vrps -o "$OUT_DIR/vrps.csv")
+JSON_ARGS=(vrps -f jsonext -o "$OUT_DIR/routinator.json")
+
+/usr/bin/time -v -o "$OUT_DIR/process-time.txt" bash -c '
+  set -euo pipefail
+  if [[ -n "$LOCAL_REPO_REPLAY_VALIDATION_TIME" && -n "${FAKETIME_LIB:-}" ]]; then
+    faketime_value="$(python3 - <<'"'"'PY'"'"' "$LOCAL_REPO_REPLAY_VALIDATION_TIME"
+from datetime import datetime, timezone
+import sys
+dt = datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc)
+print("@" + dt.strftime("%Y-%m-%d %H:%M:%S"))
+PY
+)"
+    export LD_PRELOAD="$FAKETIME_LIB"
+    export TZ=UTC
+    unset FAKETIME_FMT
+    export FAKETIME="$faketime_value"
+    export FAKETIME_DONT_FAKE_MONOTONIC=1
+  fi
+  "$@" update --complete >"$LOCAL_REPO_REPLAY_OUT_DIR/update.log" 2>&1 || true
+  "$@" vrps -o "$LOCAL_REPO_REPLAY_OUT_DIR/vrps.csv" >"$LOCAL_REPO_REPLAY_OUT_DIR/vrps.log" 2>&1
+  "$@" vrps -f jsonext -o "$LOCAL_REPO_REPLAY_OUT_DIR/routinator.json" >"$LOCAL_REPO_REPLAY_OUT_DIR/json.log" 2>&1
+' local_repo_replay "${ARGS[@]}"
+python3 "$SCRIPT_DIR/normalize_rp_outputs.py" --kind routinator --input "$OUT_DIR/routinator.json" --vrps-out "$OUT_DIR/vrps.normalized.txt" --vaps-out "$OUT_DIR/vaps.normalized.txt"
+python3 "$SCRIPT_DIR/summarize_replay.py" --rp routinator --out-dir "$OUT_DIR" --summary "$OUT_DIR/summary.json"
+echo "done: $OUT_DIR"
--- a/scripts/local_repo_replay/run_rpki_client_from_local_tree.sh
+++ b/scripts/local_repo_replay/run_rpki_client_from_local_tree.sh
@@ -0,0 +1,107 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+usage() {
+  cat <<'EOF'
+Usage:
+  run_rpki_client_from_local_tree.sh \
+    --rpki-client-bin <path> \
+    --mirror-root <local-rsync-mirror-root> \
+    --tal-dir <dir> | --tal <file>... \
+    --out-dir <path> \
+    [--cache-dir <work-cache-dir>] \
+    [--validation-time <RFC3339>] \
+    [--real-rsync-bin <path>] \
+    [--parser-workers <n>]
+
+The input mirror is prepared by the caller. This script disables RRDP, points
+rpki-client's rsync command at a local URI mapper, and fetches only from the
+local filesystem tree.
+EOF
+}
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+RPKI_CLIENT_BIN=""
+CACHE_DIR=""
+MIRROR_ROOT=""
+OUT_DIR=""
+TAL_DIR=""
+VALIDATION_TIME=""
+PARSER_WORKERS="${PARSER_WORKERS:-4}"
+REAL_RSYNC_BIN="${REAL_RSYNC_BIN:-/usr/bin/rsync}"
+TALS=()
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --rpki-client-bin) RPKI_CLIENT_BIN="$2"; shift 2 ;;
+    --mirror-root) MIRROR_ROOT="$2"; shift 2 ;;
+    --cache-dir) CACHE_DIR="$2"; shift 2 ;;
+    --tal-dir) TAL_DIR="$2"; shift 2 ;;
+    --tal) TALS+=("$2"); shift 2 ;;
+    --out-dir) OUT_DIR="$2"; shift 2 ;;
+    --validation-time) VALIDATION_TIME="$2"; shift 2 ;;
+    --real-rsync-bin) REAL_RSYNC_BIN="$2"; shift 2 ;;
+    --parser-workers) PARSER_WORKERS="$2"; shift 2 ;;
+    -h|--help) usage; exit 0 ;;
+    *) echo "unknown argument: $1" >&2; usage >&2; exit 2 ;;
+  esac
+done
+
+[[ -n "$RPKI_CLIENT_BIN" && -n "$MIRROR_ROOT" && -n "$OUT_DIR" ]] || { usage >&2; exit 2; }
+[[ -x "$RPKI_CLIENT_BIN" ]] || { echo "rpki-client binary not executable: $RPKI_CLIENT_BIN" >&2; exit 2; }
+[[ -d "$MIRROR_ROOT" ]] || { echo "mirror root not found: $MIRROR_ROOT" >&2; exit 2; }
+if [[ -z "$TAL_DIR" && "${#TALS[@]}" -eq 0 ]]; then
+  echo "provide --tal-dir or at least one --tal" >&2
+  exit 2
+fi
+
+mkdir -p "$OUT_DIR"
+if [[ -z "$CACHE_DIR" ]]; then
+  CACHE_DIR="$OUT_DIR/cache"
+fi
+mkdir -p "$CACHE_DIR" "$OUT_DIR/out"
+chmod a+rwx "$OUT_DIR" "$CACHE_DIR" "$OUT_DIR/out"
+WORK_DIR="$OUT_DIR/work"
+PREPARED_TALS="$WORK_DIR/tals"
+rm -rf "$WORK_DIR"
+mkdir -p "$PREPARED_TALS"
+PREPARE_TAL_ARGS=(--out-dir "$PREPARED_TALS" --summary "$OUT_DIR/prepared-tals.json")
+if [[ -n "$TAL_DIR" ]]; then
+  PREPARE_TAL_ARGS+=(--tal-dir "$TAL_DIR")
+fi
+for tal in "${TALS[@]}"; do
+  PREPARE_TAL_ARGS+=(--tal "$tal")
+done
+python3 "$SCRIPT_DIR/prepare_tals.py" "${PREPARE_TAL_ARGS[@]}"
+
+CLIENT_TAL_ARGS=()
+while IFS= read -r tal; do
+  CLIENT_TAL_ARGS+=(-t "$tal")
+done < <(find "$PREPARED_TALS" -maxdepth 1 -type f -name '*.tal' | sort)
+
+TIME_ARGS=()
+if [[ -n "$VALIDATION_TIME" ]]; then
+  epoch="$(python3 - <<'PY' "$VALIDATION_TIME"
+from datetime import datetime, timezone
+import sys
+print(int(datetime.fromisoformat(sys.argv[1].replace("Z", "+00:00")).astimezone(timezone.utc).timestamp()))
+PY
+)"
+  TIME_ARGS=(-P "$epoch")
+fi
+
+export CIR_MIRROR_ROOT="$(cd "$MIRROR_ROOT" && pwd)"
+export REAL_RSYNC_BIN="$REAL_RSYNC_BIN"
+export CIR_LOCAL_LINK_MODE=1
+
+/usr/bin/time -v -o "$OUT_DIR/process-time.txt" \
+  "$RPKI_CLIENT_BIN" \
+  -R -e "$SCRIPT_DIR/cir-rsync-wrapper" -j -c -p "$PARSER_WORKERS" \
+  "${TIME_ARGS[@]}" \
+  "${CLIENT_TAL_ARGS[@]}" \
+  -d "$CACHE_DIR" \
+  "$OUT_DIR/out" >"$OUT_DIR/stdout.log" 2>"$OUT_DIR/stderr.log"
+
+python3 "$SCRIPT_DIR/normalize_rp_outputs.py" --kind rpki-client --input "$OUT_DIR/out" --vrps-out "$OUT_DIR/vrps.normalized.txt" --vaps-out "$OUT_DIR/vaps.normalized.txt"
+python3 "$SCRIPT_DIR/summarize_replay.py" --rp rpki-client --out-dir "$OUT_DIR" --summary "$OUT_DIR/summary.json"
+echo "done: $OUT_DIR"
--- a/scripts/local_repo_replay/summarize_replay.py
+++ b/scripts/local_repo_replay/summarize_replay.py
@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import json
+import re
+from pathlib import Path
+
+
+def count_lines(path: Path) -> int:
+    if not path.is_file():
+        return 0
+    return sum(1 for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.strip())
+
+
+def parse_time(path: Path) -> dict[str, object]:
+    if not path.is_file():
+        return {}
+    text = path.read_text(encoding="utf-8", errors="replace")
+    elapsed = re.search(r"Elapsed \(wall clock\) time .*: (.+)", text)
+    rss = re.search(r"Maximum resident set size \(kbytes\): (\d+)", text)
+    return {
+        "elapsed": elapsed.group(1).strip() if elapsed else "",
+        "maxRssKb": int(rss.group(1)) if rss else 0,
+    }
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--rp", required=True)
+    parser.add_argument("--out-dir", type=Path, required=True)
+    parser.add_argument("--summary", type=Path, required=True)
+    args = parser.parse_args()
+    summary = {
+        "rp": args.rp,
+        "vrps": count_lines(args.out_dir / "vrps.normalized.txt"),
+        "vaps": count_lines(args.out_dir / "vaps.normalized.txt"),
+        "time": parse_time(args.out_dir / "process-time.txt"),
+        "artifacts": sorted(path.name for path in args.out_dir.iterdir() if path.is_file()),
+    }
+    args.summary.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n", encoding="utf-8")
+    print(args.summary)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
--- a/scripts/local_repo_replay/templates/README.md
+++ b/scripts/local_repo_replay/templates/README.md
@@ -0,0 +1,118 @@
+# Local Repository Tree Replay Package
+
+This package replays already prepared local RPKI repository trees with
+Routinator and rpki-client.
+
+It is intentionally independent from CIR:
+
+- it does not read `.cir`;
+- it does not read `repo-bytes.db`;
+- it does not call `cir_materialize`;
+- it does not generate a local repository tree.
+
+The caller must prepare the local repository/cache tree before running these
+scripts.
+
+## Contents
+
+```text
+local-repo-replay-package/
+  scripts/
+    run_routinator_from_local_tree.sh
+    run_rpki_client_from_local_tree.sh
+    run_dual_local_tree_replay.sh
+    prepare_tals.py
+    cir-rsync-wrapper
+    cir-local-link-sync.py
+    normalize_rp_outputs.py
+    compare_normalized_sets.py
+    summarize_replay.py
+  docs/
+    input_tree_requirements.md
+    offline_replay_limits.md
+    output_files.md
+  examples/
+    routinator_example.sh
+    rpki_client_example.sh
+    dual_compare_example.sh
+  env.example
+```
+
+## Routinator replay
+
+```bash
+./scripts/run_routinator_from_local_tree.sh \
+  --routinator-bin /opt/routinator/target/release/routinator \
+  --mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/routinator \
+  --enable-aspa
+```
+
+The script uses `--disable-rrdp`, `--rsync-command ./scripts/cir-rsync-wrapper`,
+and the local mirror root to satisfy rsync fetches from the local filesystem.
+The wrapper name is historical; in this package it is only a generic
+`rsync://` to local-path mapper.
+
+If `--validation-time` is needed for Routinator, set `FAKETIME_LIB` to a working
+libfaketime shared library. Otherwise Routinator validates at wall-clock time.
+
+On Ubuntu, install and use faketime like this:
+
+```bash
+sudo apt-get install -y libfaketime
+export FAKETIME_LIB=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1
+./scripts/run_routinator_from_local_tree.sh \
+  --routinator-bin /opt/routinator/target/release/routinator \
+  --mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/routinator \
+  --validation-time 2026-05-14T06:48:00Z \
+  --enable-aspa
+```
+
+Without `FAKETIME_LIB`, old local trees can produce empty or smaller output
+because Routinator validates manifests and CRLs against current wall-clock time.
+
+## rpki-client replay
+
+```bash
+./scripts/run_rpki_client_from_local_tree.sh \
+  --rpki-client-bin /opt/rpki-client/src/rpki-client \
+  --mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/rpki-client \
+  --parser-workers 4
+```
+
+The script uses `rpki-client -R -e ./scripts/cir-rsync-wrapper` so RRDP is
+disabled and rsync fetches are served from the local mirror. `--cache-dir` is an
+optional working cache directory used by rpki-client during this local replay.
+
+## Dual replay
+
+```bash
+./scripts/run_dual_local_tree_replay.sh \
+  --routinator-bin /opt/routinator/target/release/routinator \
+  --routinator-mirror-root /data/replay/mirror \
+  --rpki-client-bin /opt/rpki-client/src/rpki-client \
+  --rpki-client-mirror-root /data/replay/mirror \
+  --tal-dir /data/replay/tals \
+  --out-dir /data/replay/out/dual
+```
+
+If `--validation-time` is passed to dual replay, remember to export
+`FAKETIME_LIB` first so Routinator and rpki-client use the same logical
+validation time.
+
+## Outputs
+
+Each run writes normalized output:
+
+- `vrps.normalized.txt`
+- `vaps.normalized.txt`
+- `summary.json`
+- raw RP output and logs
+- `process-time.txt`
+
+See `docs/output_files.md`.
--- a/scripts/local_repo_replay/templates/docs/input_tree_requirements.md
+++ b/scripts/local_repo_replay/templates/docs/input_tree_requirements.md
@@ -0,0 +1,39 @@
+# Input Tree Requirements
+
+The input tree is not part of this package. The caller must prepare it before
+running replay.
+
+## Mirror root
+
+The mirror root must map rsync URIs to local paths:
+
+```text
+rsync://rpki.example.net/repo/a/b/c.roa
+=> <mirror-root>/rpki.example.net/repo/a/b/c.roa
+```
+
+The tree must contain all objects needed by the selected TALs: TA certificates,
+manifests, CRLs, ROAs, ASPAs, router certs, and child CA certificates.
+
+Both Routinator and rpki-client scripts consume this same mirror root through a
+local rsync command wrapper.
+
+## rpki-client working cache
+
+For rpki-client replay, `--cache-dir` is only rpki-client's working cache
+directory for this local run. It is not the input dataset. The authoritative
+input is `--mirror-root`.
+
+## TALs
+
+Provide either `--tal-dir <dir>` or repeated `--tal <file>`.
+
+The scripts prepare a replay-local TAL copy that prefers `rsync://` TA
+certificate URIs. This prevents a TAL with an HTTPS URI listed first from
+escaping to the network during local replay. The TAL set should match the local
+tree. Mixing a tree from one run with different TALs may produce meaningless
+differences.
+
+## No generation
+
+This package does not generate the tree and does not repair missing objects.
--- a/scripts/local_repo_replay/templates/docs/offline_replay_limits.md
+++ b/scripts/local_repo_replay/templates/docs/offline_replay_limits.md
@@ -0,0 +1,38 @@
+# Offline Replay Limits
+
+## Routinator
+
+The Routinator script disables RRDP and uses an rsync command wrapper to map
+rsync URLs to local paths. It still runs Routinator's normal validation logic.
+
+If the local mirror does not contain required objects, validation can fail or
+produce fewer outputs.
+
+## rpki-client
+
+The rpki-client script uses `-R` to disable RRDP and `-e` to point rsync at the
+local mapper. rpki-client still builds its normal working cache, but every
+rsync source is rewritten to the local mirror.
+
+If the mirror was incomplete or produced by a different TAL set, replay results
+may differ from the original run.
+
+## Validation time
+
+rpki-client supports `-P <posix-seconds>`. Routinator does not expose the same
+simple command-line evaluation-time option in the tested version; if `FAKETIME_LIB`
+is configured, the script can run Routinator under faketime. Without
+`FAKETIME_LIB`, `--validation-time` is intentionally ignored for Routinator and
+current wall-clock validation can reject stale manifests or CRLs.
+
+Ubuntu example:
+
+```bash
+sudo apt-get install -y libfaketime
+export FAKETIME_LIB=/usr/lib/x86_64-linux-gnu/faketime/libfaketime.so.1
+```
+
+The script sets `TZ=UTC` and converts RFC3339 validation time to libfaketime
+absolute UTC format, for example `2026-05-14T06:48:00Z` becomes
+`@2026-05-14 06:48:00`. Setting `TZ=UTC` is required because libfaketime parses
+absolute timestamps in the process timezone.
--- a/scripts/local_repo_replay/templates/docs/output_files.md
+++ b/scripts/local_repo_replay/templates/docs/output_files.md
@@ -0,0 +1,16 @@
+# Output Files
+
+Each replay output directory can contain:
+
+- `vrps.normalized.txt`: one normalized VRP per line.
+- `vaps.normalized.txt`: one normalized VAP/ASPA per line.
+- `summary.json`: counts and resource summary.
+- `process-time.txt`: `/usr/bin/time -v` output.
+- RP-specific raw output:
+  - Routinator: `routinator.json`, `vrps.csv`.
+  - rpki-client: `out/json`, `out/csv`, and other native files.
+- logs:
+  - `stdout.log` / `stderr.log` for rpki-client.
+  - `update.log`, `vrps.log`, `json.log` for Routinator.
+
+Dual replay additionally writes `compare-summary.json`.
--- a/scripts/local_repo_replay/templates/examples/dual_compare_example.sh
+++ b/scripts/local_repo_replay/templates/examples/dual_compare_example.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+./scripts/run_dual_local_tree_replay.sh \
+  --routinator-bin "${ROUTINATOR_BIN:-/opt/routinator/target/release/routinator}" \
+  --routinator-mirror-root "${ROUTINATOR_MIRROR_ROOT:-/data/replay/mirror}" \
+  --rpki-client-bin "${RPKI_CLIENT_BIN:-/opt/rpki-client/src/rpki-client}" \
+  --rpki-client-mirror-root "${RPKI_CLIENT_MIRROR_ROOT:-/data/replay/mirror}" \
+  --rpki-client-cache-dir "${RPKI_CLIENT_CACHE_DIR:-/data/replay/work/rpki-client-cache}" \
+  --tal-dir "${TAL_DIR:-/data/replay/tals}" \
+  --out-dir "${OUT_DIR:-/data/replay/out/dual}"
--- a/scripts/local_repo_replay/templates/examples/routinator_example.sh
+++ b/scripts/local_repo_replay/templates/examples/routinator_example.sh
@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+./scripts/run_routinator_from_local_tree.sh \
+  --routinator-bin "${ROUTINATOR_BIN:-/opt/routinator/target/release/routinator}" \
+  --mirror-root "${MIRROR_ROOT:-/data/replay/mirror}" \
+  --tal-dir "${TAL_DIR:-/data/replay/tals}" \
+  --out-dir "${OUT_DIR:-/data/replay/out/routinator}" \
+  --enable-aspa
--- a/scripts/local_repo_replay/templates/examples/rpki_client_example.sh
+++ b/scripts/local_repo_replay/templates/examples/rpki_client_example.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+./scripts/run_rpki_client_from_local_tree.sh \
+  --rpki-client-bin "${RPKI_CLIENT_BIN:-/opt/rpki-client/src/rpki-client}" \
+  --mirror-root "${MIRROR_ROOT:-/data/replay/mirror}" \
+  --cache-dir "${RPKI_CLIENT_CACHE_DIR:-/data/replay/work/rpki-client-cache}" \
+  --tal-dir "${TAL_DIR:-/data/replay/tals}" \
+  --out-dir "${OUT_DIR:-/data/replay/out/rpki-client}" \
+  --parser-workers "${PARSER_WORKERS:-4}"
--- a/scripts/soak/portable-soak.env.example
+++ b/scripts/soak/portable-soak.env.example
@@ -2,6 +2,7 @@
 # 复制为 .env 后可以在远端直接调整；所有路径默认相对 package 根目录。

 # 最大运行轮次。重复执行 run_soak.sh 时会从已有最后一轮之后继续编号。
+# 正整数表示固定运行轮次；负数（例如 -1）表示持续运行不自动停止；0 非法。
 MAX_RUNS=3

 # 两轮之间等待秒数。做连续无等待验收时设置为 0。
@@ -15,7 +16,7 @@ RIRS=afrinic,apnic,arin,lacnic,ripe
 # 运行根目录。默认使用 package 根目录；如需把产物写到独立数据盘，可改成绝对路径。
 RUN_ROOT="${PACKAGE_ROOT}"

-# 保留最近多少轮 run 目录。旧 run 会由 rpki_daemon 自身或后续脚本策略清理。
+# 保留最近多少轮 run 目录。持续运行模式建议设置为 5 或按磁盘容量评估。
 RETAIN_RUNS=10

 # 是否输出 compact report JSON。1 表示启用，0 表示关闭。
--- a/scripts/soak/run_soak.sh
+++ b/scripts/soak/run_soak.sh
@@ -77,6 +77,11 @@ validate_non_negative_int() {
  [[ "$value" =~ ^[0-9]+$ ]] || die "$name must be an integer: $value"
 }

+validate_max_runs() {
+  [[ "$MAX_RUNS" =~ ^-?[0-9]+$ ]] || die "MAX_RUNS must be an integer: $MAX_RUNS"
+  [[ "$MAX_RUNS" != "0" ]] || die "MAX_RUNS must be non-zero; use a positive value for fixed runs or -1 for continuous mode"
+}
+
 validate_rsync_scope() {
  case "$RSYNC_SCOPE" in
    publication-point|module-root)
@@ -486,20 +491,30 @@ PY

 apply_outer_retention() {
  local dirs=()
+  local retain_limit="$RETAIN_RUNS"
+  local keep_run="${1:-}"
  local run_dir
  shopt -s nullglob
  for run_dir in "$RUNS_ROOT"/run_[0-9][0-9][0-9][0-9]; do
    [[ -d "$run_dir" ]] && dirs+=("$run_dir")
  done
  shopt -u nullglob
-  if (( ${#dirs[@]} <= RETAIN_RUNS )); then
+  if (( ${#dirs[@]} <= retain_limit )); then
    return 0
  fi
  mapfile -t dirs < <(printf '%s\n' "${dirs[@]}" | sort)
-  local remove_count=$(( ${#dirs[@]} - RETAIN_RUNS ))
-  local index
-  for (( index = 0; index < remove_count; index++ )); do
-    rm -rf "${dirs[$index]}"
+  local remove_count=$(( ${#dirs[@]} - retain_limit ))
+  local removed_count=0
+  local candidate
+  for candidate in "${dirs[@]}"; do
+    if [[ -n "$keep_run" && "$(basename "$candidate")" == "$keep_run" ]]; then
+      continue
+    fi
+    rm -rf "$candidate"
+    removed_count=$((removed_count + 1))
+    if (( removed_count >= remove_count )); then
+      break
+    fi
  done
 }

@@ -519,6 +534,7 @@ run_one_round() {
  local summary_state

  mkdir -p "$run_dir" "$daemon_state_root"
+  apply_outer_retention "$run_id"
  started_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  write_run_meta "$run_dir/run-meta.json" "running" "$run_index" "$run_id" "$sync_mode" \
    "$snapshot_reason" "$previous_run_id" "$previous_success_value" "$started_at" "" \
@@ -573,7 +589,7 @@ main() {
  require_command python3
  require_command date
  require_command find
-  validate_positive_int "MAX_RUNS" "$MAX_RUNS"
+  validate_max_runs
  validate_non_negative_int "INTERVAL_SECS" "$INTERVAL_SECS"
  validate_positive_int "RETAIN_RUNS" "$RETAIN_RUNS"
  validate_rsync_scope
@@ -599,12 +615,20 @@ main() {

  local max_index
  local next_index
+  local run_forever=0
+  local stop_index=0
  max_index="$(max_existing_run_index)"
  next_index=$((max_index + 1))
-  local stop_index=$((max_index + MAX_RUNS))
+  if (( MAX_RUNS < 0 )); then
+    run_forever=1
+    echo "run_soak mode=continuous max_existing_run_index=$max_index next_run=$(printf 'run_%04d' "$next_index")"
+  else
+    stop_index=$((max_index + MAX_RUNS))
+    echo "run_soak mode=fixed max_existing_run_index=$max_index next_run=$(printf 'run_%04d' "$next_index") stop_run=$(printf 'run_%04d' "$stop_index")"
+  fi
  local any_failed=0

-  while (( next_index <= stop_index )); do
+  while (( run_forever == 1 || next_index <= stop_index )); do
    INVALID_DB_PATH=""
    INVALID_STATE_PATH=""
    INVALID_TMP_PATH=""
@@ -649,7 +673,7 @@ main() {
      echo "completed run $(printf 'run_%04d' "$next_index") status=failed" >&2
      any_failed=1
    fi
-    if (( next_index < stop_index && INTERVAL_SECS > 0 )); then
+    if (( (run_forever == 1 || next_index < stop_index) && INTERVAL_SECS > 0 )); then
      sleep "$INTERVAL_SECS"
    fi
    next_index=$((next_index + 1))