[#49] swarm test重启测试通过

src/bundle/cpu-node-bundle/Dockerfile

@@ -19,15 +19,15 @@ WORKDIR /
 
 # Offline fluent-bit assets and bundle tarball are staged by the build script
 COPY node-bootstrap.sh /usr/local/bin/node-bootstrap.sh
+COPY health-watcher.sh /usr/local/bin/health-watcher.sh
 COPY private/start-fluent-bit.sh /private/start-fluent-bit.sh
 COPY private/etc /private/etc
 COPY private/packages /private/packages
 COPY bundle/ /bundle/
 
-RUN chmod +x /usr/local/bin/node-bootstrap.sh /private/start-fluent-bit.sh || true; \
+RUN chmod +x /usr/local/bin/node-bootstrap.sh /usr/local/bin/health-watcher.sh /private/start-fluent-bit.sh || true; \
     mkdir -p /logs/train /logs/infer /buffers /opt/argus-metric; \
     if [ "${ARGUS_LOGS_WORLD_WRITABLE}" = "1" ]; then chmod 1777 /logs/train /logs/infer || true; else chmod 755 /logs/train /logs/infer || true; fi; \
     chmod 770 /buffers || true
 
 ENTRYPOINT ["/usr/local/bin/node-bootstrap.sh"]
-

src/bundle/cpu-node-bundle/health-watcher.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# health-watcher.sh (CPU node bundle)
+# 周期执行 check_health.sh 与 restart_unhealthy.sh，用于节点容器内自愈。
+
+INSTALL_ROOT="/opt/argus-metric"
+INTERVAL="${HEALTH_WATCH_INTERVAL:-60}"
+VER_DIR="${1:-}"
+
+log(){ echo "[HEALTH-WATCHER] $*"; }
+
+resolve_ver_dir() {
+  local dir=""
+  if [[ -n "${VER_DIR:-}" && -d "$VER_DIR" ]]; then
+    dir="$VER_DIR"
+  elif [[ -L "$INSTALL_ROOT/current" ]]; then
+    dir="$(readlink -f "$INSTALL_ROOT/current" 2>/dev/null || true)"
+  fi
+  if [[ -z "$dir" ]]; then
+    dir="$(ls -d "$INSTALL_ROOT"/versions/* 2>/dev/null | sort -V | tail -n1 || true)"
+  fi
+  echo "$dir"
+}
+
+main() {
+  log "starting with interval=${INTERVAL}s"
+  local dir
+  dir="$(resolve_ver_dir)"
+  if [[ -z "$dir" || ! -d "$dir" ]]; then
+    log "no valid install dir found under $INSTALL_ROOT; exiting"
+    exit 0
+  fi
+
+  local chk="$dir/check_health.sh"
+  local rst="$dir/restart_unhealthy.sh"
+
+  if [[ ! -x "$chk" && ! -x "$rst" ]]; then
+    log "neither check_health.sh nor restart_unhealthy.sh is executable under $dir; exiting"
+    exit 0
+  fi
+
+  log "watching install dir: $dir"
+
+  while :; do
+    if [[ -x "$chk" ]]; then
+      log "running check_health.sh"
+      "$chk" >> "$dir/.health_check.watch.log" 2>&1 || log "check_health.sh reported issues (see .health_check.watch.log)"
+    fi
+    if [[ -x "$rst" ]]; then
+      log "running restart_unhealthy.sh"
+      "$rst" >> "$dir/.restart.watch.log" 2>&1 || log "restart_unhealthy.sh reported issues (see .restart.watch.log)"
+    fi
+    sleep "$INTERVAL"
+  done
+}
+
+main "$@"
+

src/bundle/cpu-node-bundle/node-bootstrap.sh

@@ -119,6 +119,13 @@ for i in {1..60}; do
   sleep 2
 done
 
+# 6) spawn health watcher (best-effort, non-blocking)
+if command -v /usr/local/bin/health-watcher.sh >/dev/null 2>&1; then
+  echo "[BOOT] starting health watcher for $ver_dir"
+  setsid /usr/local/bin/health-watcher.sh "${ver_dir:-}" >/var/log/health-watcher.log 2>&1 < /dev/null || true &
+else
+  echo "[BOOT][WARN] health-watcher.sh not found; skip health watcher"
+fi
+
 echo "[BOOT] ready; entering sleep"
 exec sleep infinity
-

src/bundle/gpu-node-bundle/Dockerfile

@@ -31,11 +31,12 @@ WORKDIR /
 # Expect staged build context to provide these directories/files
 COPY bundle/ /bundle/
 COPY node-bootstrap.sh /usr/local/bin/node-bootstrap.sh
+COPY health-watcher.sh /usr/local/bin/health-watcher.sh
 COPY private/start-fluent-bit.sh /private/start-fluent-bit.sh
 COPY private/etc /private/etc
 COPY private/packages /private/packages
 
-RUN chmod +x /usr/local/bin/node-bootstrap.sh /private/start-fluent-bit.sh || true; \
+RUN chmod +x /usr/local/bin/node-bootstrap.sh /usr/local/bin/health-watcher.sh /private/start-fluent-bit.sh || true; \
     mkdir -p /logs/train /logs/infer /buffers /opt/argus-metric; \
     chmod 1777 /logs/train /logs/infer || true; \
     chmod 770 /buffers || true

src/bundle/gpu-node-bundle/health-watcher.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# health-watcher.sh (GPU bundle)
+# 周期执行 check_health.sh 与 restart_unhealthy.sh，用于 GPU 节点容器内自愈。
+
+INSTALL_ROOT="/opt/argus-metric"
+INTERVAL="${HEALTH_WATCH_INTERVAL:-60}"
+VER_DIR="${1:-}"
+
+log(){ echo "[HEALTH-WATCHER] $*"; }
+
+resolve_ver_dir() {
+  local dir=""
+  if [[ -n "${VER_DIR:-}" && -d "$VER_DIR" ]]; then
+    dir="$VER_DIR"
+  elif [[ -L "$INSTALL_ROOT/current" ]]; then
+    dir="$(readlink -f "$INSTALL_ROOT/current" 2>/dev/null || true)"
+  fi
+  if [[ -z "$dir" ]]; then
+    dir="$(ls -d "$INSTALL_ROOT"/versions/* 2>/dev/null | sort -V | tail -n1 || true)"
+  fi
+  echo "$dir"
+}
+
+main() {
+  log "starting with interval=${INTERVAL}s"
+  local dir
+  dir="$(resolve_ver_dir)"
+  if [[ -z "$dir" || ! -d "$dir" ]]; then
+    log "no valid install dir found under $INSTALL_ROOT; exiting"
+    exit 0
+  fi
+
+  local chk="$dir/check_health.sh"
+  local rst="$dir/restart_unhealthy.sh"
+
+  if [[ ! -x "$chk" && ! -x "$rst" ]]; then
+    log "neither check_health.sh nor restart_unhealthy.sh is executable under $dir; exiting"
+    exit 0
+  fi
+
+  log "watching install dir: $dir"
+
+  while :; do
+    if [[ -x "$chk" ]]; then
+      log "running check_health.sh"
+      "$chk" >> "$dir/.health_check.watch.log" 2>&1 || log "check_health.sh reported issues (see .health_check.watch.log)"
+    fi
+    if [[ -x "$rst" ]]; then
+      log "running restart_unhealthy.sh"
+      "$rst" >> "$dir/.restart.watch.log" 2>&1 || log "restart_unhealthy.sh reported issues (see .restart.watch.log)"
+    fi
+    sleep "$INTERVAL"
+  done
+}
+
+main "$@"
+

src/bundle/gpu-node-bundle/node-bootstrap.sh

@@ -123,5 +123,13 @@ for i in {1..60}; do
   sleep 2
 done
 
+# 6) spawn health watcher (best-effort, non-blocking)
+if command -v /usr/local/bin/health-watcher.sh >/dev/null 2>&1; then
+  echo "[BOOT] starting health watcher for $ver_dir"
+  setsid /usr/local/bin/health-watcher.sh "${ver_dir:-}" >/var/log/health-watcher.log 2>&1 < /dev/null || true &
+else
+  echo "[BOOT][WARN] health-watcher.sh not found; skip health watcher"
+fi
+
 echo "[BOOT] ready; entering sleep"
 exec sleep infinity