20260514 rsync默认限定发布点scope

2026-05-15 17:17:41 +08:00
parent 137b3516d0
commit 615f8709af
4 changed files with 196 additions and 10 deletions
--- a/scripts/soak/portable-soak.env.example
+++ b/scripts/soak/portable-soak.env.example
@@ -24,6 +24,10 @@ OUTPUT_COMPACT_REPORT=1
 # 是否复用持久 rsync mirror。1 表示跨 run 复用；失败隔离数据库时也不会清理 mirror。
 ALLOW_RSYNC_MIRROR_REUSE=1

+# rsync 同步 scope。
+# publication-point 表示默认只同步当前发布点；module-root 表示扩大到 rsync module 根目录。
+RSYNC_SCOPE=publication-point
+
 # 前一轮失败或不完整时，是否隔离旧数据库和运行态目录后强制下一轮 snapshot。
 # 建议保持 1；设置为 0 时，检测到前一轮失败会直接停止。
 FAILURE_SNAPSHOT_RESET=1
--- a/scripts/soak/run_soak.sh
+++ b/scripts/soak/run_soak.sh
@@ -17,6 +17,7 @@ RUN_ROOT="${RUN_ROOT:-$PACKAGE_ROOT}"
 RETAIN_RUNS="${RETAIN_RUNS:-10}"
 OUTPUT_COMPACT_REPORT="${OUTPUT_COMPACT_REPORT:-1}"
 ALLOW_RSYNC_MIRROR_REUSE="${ALLOW_RSYNC_MIRROR_REUSE:-1}"
+RSYNC_SCOPE="${RSYNC_SCOPE:-publication-point}"
 FAILURE_SNAPSHOT_RESET="${FAILURE_SNAPSHOT_RESET:-1}"
 DB_STATS_EXACT_EVERY="${DB_STATS_EXACT_EVERY:-3}"
 RPKI_PROGRESS_LOG="${RPKI_PROGRESS_LOG:-1}"
@@ -76,6 +77,16 @@ validate_non_negative_int() {
  [[ "$value" =~ ^[0-9]+$ ]] || die "$name must be an integer: $value"
 }

+validate_rsync_scope() {
+  case "$RSYNC_SCOPE" in
+    publication-point|module-root)
+      ;;
+    *)
+      die "RSYNC_SCOPE must be publication-point or module-root: $RSYNC_SCOPE"
+      ;;
+  esac
+}
+
 normalize_token() {
  local token="$1"
  token="${token#"${token%%[![:space:]]*}"}"
@@ -335,6 +346,7 @@ build_child_args() {
  CHILD_ARGS=(
    --db "$DB_DIR/work-db"
    --repo-bytes-db "$DB_DIR/repo-bytes.db"
+    --rsync-scope "$RSYNC_SCOPE"
  )
  if is_true "$ALLOW_RSYNC_MIRROR_REUSE"; then
    CHILD_ARGS+=(--rsync-mirror-root "$RSYNC_MIRROR_ROOT")
@@ -382,6 +394,8 @@ build_child_args() {
 copy_inner_run_outputs() {
  local daemon_state_root="$1"
  local run_dir="$2"
+  local outer_run_index="$3"
+  local outer_run_id="$4"
  local inner_run_dir
  inner_run_dir="$(find "$daemon_state_root/runs" -mindepth 1 -maxdepth 1 -type d 2>/dev/null | sort | tail -n 1 || true)"
  if [[ -n "$inner_run_dir" && -d "$inner_run_dir" ]]; then
@@ -389,8 +403,85 @@ copy_inner_run_outputs() {
    cp -a "$inner_run_dir"/. "$run_dir"/
    shopt -u dotglob nullglob
  fi
-  [[ -f "$daemon_state_root/daemon-status.json" ]] && cp "$daemon_state_root/daemon-status.json" "$run_dir/daemon-status.json"
-  [[ -f "$daemon_state_root/daemon-runs.jsonl" ]] && cp "$daemon_state_root/daemon-runs.jsonl" "$run_dir/daemon-runs.jsonl"
+  [[ -f "$daemon_state_root/daemon-status.json" ]] && cp "$daemon_state_root/daemon-status.json" "$run_dir/daemon-status.inner.json"
+  [[ -f "$daemon_state_root/daemon-runs.jsonl" ]] && cp "$daemon_state_root/daemon-runs.jsonl" "$run_dir/daemon-runs.inner.jsonl"
+  normalize_outer_run_metadata "$run_dir" "$outer_run_index" "$outer_run_id" "$inner_run_dir" "$daemon_state_root"
+}
+
+normalize_outer_run_metadata() {
+  local run_dir="$1"
+  local outer_run_index="$2"
+  local outer_run_id="$3"
+  local inner_run_dir="$4"
+  local daemon_state_root="$5"
+  python3 - "$run_dir" "$outer_run_index" "$outer_run_id" "$inner_run_dir" "$daemon_state_root" <<'PY'
+import json
+import pathlib
+import sys
+
+run_dir = pathlib.Path(sys.argv[1]).resolve()
+outer_run_index = int(sys.argv[2])
+outer_run_id = sys.argv[3]
+inner_run_dir = sys.argv[4]
+daemon_state_root = pathlib.Path(sys.argv[5])
+
+def replace_paths(value):
+    if isinstance(value, dict):
+        return {key: replace_paths(item) for key, item in value.items()}
+    if isinstance(value, list):
+        return [replace_paths(item) for item in value]
+    if isinstance(value, str) and inner_run_dir:
+        return value.replace(inner_run_dir, str(run_dir))
+    return value
+
+def normalize_summary(summary):
+    summary = dict(summary)
+    summary.setdefault("innerRunSeq", summary.get("runSeq"))
+    summary.setdefault("innerRunId", summary.get("runId"))
+    summary.setdefault("innerRunDir", summary.get("runDir"))
+    summary = replace_paths(summary)
+    summary["runSeq"] = outer_run_index
+    summary["runId"] = outer_run_id
+    summary["runDir"] = str(run_dir)
+    return summary
+
+summary_path = run_dir / "run-summary.json"
+if summary_path.exists():
+    summary = json.loads(summary_path.read_text(encoding="utf-8"))
+    summary_path.write_text(
+        json.dumps(normalize_summary(summary), indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+
+inner_status_path = run_dir / "daemon-status.inner.json"
+if not inner_status_path.exists():
+    raw_status_path = daemon_state_root / "daemon-status.json"
+    if raw_status_path.exists():
+        inner_status_path.write_text(raw_status_path.read_text(encoding="utf-8"), encoding="utf-8")
+if inner_status_path.exists():
+    status = json.loads(inner_status_path.read_text(encoding="utf-8"))
+    status.setdefault("innerLastRunId", status.get("lastRunId"))
+    status["lastRunId"] = outer_run_id
+    status["outerRunId"] = outer_run_id
+    status["outerRunIndex"] = outer_run_index
+    (run_dir / "daemon-status.json").write_text(
+        json.dumps(status, indent=2, sort_keys=True) + "\n",
+        encoding="utf-8",
+    )
+
+inner_runs_path = run_dir / "daemon-runs.inner.jsonl"
+if not inner_runs_path.exists():
+    raw_runs_path = daemon_state_root / "daemon-runs.jsonl"
+    if raw_runs_path.exists():
+        inner_runs_path.write_text(raw_runs_path.read_text(encoding="utf-8"), encoding="utf-8")
+if inner_runs_path.exists():
+    lines = []
+    for line in inner_runs_path.read_text(encoding="utf-8").splitlines():
+        if not line.strip():
+            continue
+        lines.append(json.dumps(normalize_summary(json.loads(line)), sort_keys=True))
+    (run_dir / "daemon-runs.jsonl").write_text("\n".join(lines) + ("\n" if lines else ""), encoding="utf-8")
+PY
 }

 apply_outer_retention() {
@@ -459,7 +550,7 @@ run_one_round() {
  daemon_exit_code=$?
  set -e

-  copy_inner_run_outputs "$daemon_state_root" "$run_dir"
+  copy_inner_run_outputs "$daemon_state_root" "$run_dir" "$run_index" "$run_id"
  completed_at="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
  summary_state="$(summary_status "$run_dir/run-summary.json")"
  local final_status="failed"
@@ -485,6 +576,7 @@ main() {
  validate_positive_int "MAX_RUNS" "$MAX_RUNS"
  validate_non_negative_int "INTERVAL_SECS" "$INTERVAL_SECS"
  validate_positive_int "RETAIN_RUNS" "$RETAIN_RUNS"
+  validate_rsync_scope
  if [[ -n "${DB_STATS_EXACT_EVERY:-}" && "$DB_STATS_EXACT_EVERY" != "0" ]]; then
    validate_positive_int "DB_STATS_EXACT_EVERY" "$DB_STATS_EXACT_EVERY"
  fi