完成H20服务器部署及重启测试 (#51)
当前部署情况 - h1: 部署server & client - h2: 部署client - 部署2025-11-25 - 部署目录： /home2/argus/server , /home2/argus/client - 部署使用账号：argus 网络拓扑： - h1 作为docker swarm manager - h2 作为worker加入docker swarm - docker swarm 上创建overlay network 访问方式： - 通过ssh到h1服务器，端口转发 20006-20011 端口到笔记本本地； - 门户网址：http://localhost:20006/dashboard 部署截图：    注意事项： - server各容器使用域名作为overlay network上alias别名，实现域名访问，当前版本禁用bind作为域名解析，原因是容器重启后IP变化场景bind机制复杂且不稳定。 - client 构建是内置安装包，容器启动时执行安装流程，后续重启容器跳过安装步骤。 - UID/GID：部署使用 argus账号 uid=2133, gid=2015。 Reviewed-on: #51 Reviewed-by: sundapeng <sundp@mail.zgclab.edu.cn> Reviewed-by: xuxt <xuxt@zgclab.edu.cn> Reviewed-by: huhy <husteryezi@163.com>
This commit was merged in pull request #51.
This commit is contained in:
1
src/sys/build/node-bundle/.gitignore
vendored
Normal file
1
src/sys/build/node-bundle/.gitignore
vendored
Normal file
@@ -0,0 +1 @@
|
||||
bundle/*.tar.gz
|
||||
17
src/sys/build/node-bundle/Dockerfile
Normal file
17
src/sys/build/node-bundle/Dockerfile
Normal file
@@ -0,0 +1,17 @@
|
||||
ARG BASE_IMAGE=argus-sys-metric-test-node:latest
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ARG CLIENT_VER
|
||||
LABEL org.opencontainers.image.title="argus-sys-metric-test-node-bundle" \
|
||||
org.opencontainers.image.version="${CLIENT_VER}" \
|
||||
org.opencontainers.image.description="Metric test node with embedded client package"
|
||||
|
||||
WORKDIR /
|
||||
|
||||
# bundle files are provided at build time into ./bundle in build context
|
||||
COPY bundle/ /bundle/
|
||||
COPY node-bootstrap.sh /usr/local/bin/node-bootstrap.sh
|
||||
COPY health-watcher.sh /usr/local/bin/health-watcher.sh
|
||||
RUN chmod +x /usr/local/bin/node-bootstrap.sh /usr/local/bin/health-watcher.sh
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/node-bootstrap.sh"]
|
||||
1006
src/sys/build/node-bundle/bundle/setup.sh
Executable file
1006
src/sys/build/node-bundle/bundle/setup.sh
Executable file
File diff suppressed because it is too large
Load Diff
59
src/sys/build/node-bundle/health-watcher.sh
Normal file
59
src/sys/build/node-bundle/health-watcher.sh
Normal file
@@ -0,0 +1,59 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# health-watcher.sh
|
||||
# 周期执行 check_health.sh 与 restart_unhealthy.sh，用于容器内节点自愈。
|
||||
|
||||
INSTALL_ROOT="/opt/argus-metric"
|
||||
INTERVAL="${HEALTH_WATCH_INTERVAL:-60}"
|
||||
VER_DIR="${1:-}"
|
||||
|
||||
log(){ echo "[HEALTH-WATCHER] $*"; }
|
||||
|
||||
resolve_ver_dir() {
|
||||
local dir=""
|
||||
if [[ -n "${VER_DIR:-}" && -d "$VER_DIR" ]]; then
|
||||
dir="$VER_DIR"
|
||||
elif [[ -L "$INSTALL_ROOT/current" ]]; then
|
||||
dir="$(readlink -f "$INSTALL_ROOT/current" 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ -z "$dir" ]]; then
|
||||
dir="$(ls -d "$INSTALL_ROOT"/versions/* 2>/dev/null | sort -V | tail -n1 || true)"
|
||||
fi
|
||||
echo "$dir"
|
||||
}
|
||||
|
||||
main() {
|
||||
log "starting with interval=${INTERVAL}s"
|
||||
local dir
|
||||
dir="$(resolve_ver_dir)"
|
||||
if [[ -z "$dir" || ! -d "$dir" ]]; then
|
||||
log "no valid install dir found under $INSTALL_ROOT; exiting"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
local chk="$dir/check_health.sh"
|
||||
local rst="$dir/restart_unhealthy.sh"
|
||||
|
||||
if [[ ! -x "$chk" && ! -x "$rst" ]]; then
|
||||
log "neither check_health.sh nor restart_unhealthy.sh is executable under $dir; exiting"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
log "watching install dir: $dir"
|
||||
|
||||
while :; do
|
||||
if [[ -x "$chk" ]]; then
|
||||
log "running check_health.sh"
|
||||
"$chk" >> "$dir/.health_check.watch.log" 2>&1 || log "check_health.sh reported issues (see .health_check.watch.log)"
|
||||
fi
|
||||
if [[ -x "$rst" ]]; then
|
||||
log "running restart_unhealthy.sh"
|
||||
"$rst" >> "$dir/.restart.watch.log" 2>&1 || log "restart_unhealthy.sh reported issues (see .restart.watch.log)"
|
||||
fi
|
||||
sleep "$INTERVAL"
|
||||
done
|
||||
}
|
||||
|
||||
main "$@"
|
||||
|
||||
135
src/sys/build/node-bundle/node-bootstrap.sh
Normal file
135
src/sys/build/node-bundle/node-bootstrap.sh
Normal file
@@ -0,0 +1,135 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "[BOOT] node bundle starting"
|
||||
|
||||
INSTALL_DIR="/opt/argus-metric"
|
||||
BUNDLE_DIR="/bundle"
|
||||
installed_ok=0
|
||||
|
||||
# 1) already installed?
|
||||
if [[ -L "$INSTALL_DIR/current" && -d "$INSTALL_DIR/current" ]]; then
|
||||
echo "[BOOT] client already installed at $INSTALL_DIR/current"
|
||||
else
|
||||
# 2) try local bundle first (replicate setup.sh layout: move to /opt/argus-metric/versions/<ver> and run install.sh)
|
||||
tarball=$(ls -1 "$BUNDLE_DIR"/argus-metric_*.tar.gz 2>/dev/null | head -1 || true)
|
||||
if [[ -n "${tarball:-}" ]]; then
|
||||
echo "[BOOT] installing from local bundle: $(basename "$tarball")"
|
||||
tmp=$(mktemp -d)
|
||||
tar -xzf "$tarball" -C "$tmp"
|
||||
# locate root containing version.json
|
||||
root="$tmp"
|
||||
if [[ ! -f "$root/version.json" ]]; then
|
||||
sub=$(find "$tmp" -mindepth 1 -maxdepth 1 -type d | head -n1 || true)
|
||||
[[ -n "$sub" && -f "$sub/version.json" ]] && root="$sub"
|
||||
fi
|
||||
if [[ ! -f "$root/version.json" ]]; then
|
||||
echo "[BOOT][WARN] version.json not found in bundle; fallback to FTP"
|
||||
else
|
||||
ver=$(sed -n 's/.*"version"\s*:\s*"\([^"]\+\)".*/\1/p' "$root/version.json" | head -n1)
|
||||
if [[ -z "$ver" ]]; then
|
||||
echo "[BOOT][WARN] failed to parse version from version.json; fallback to FTP"
|
||||
else
|
||||
target_root="/opt/argus-metric"
|
||||
version_dir="$target_root/versions/$ver"
|
||||
mkdir -p "$version_dir"
|
||||
# move contents into version dir
|
||||
shopt -s dotglob
|
||||
mv "$root"/* "$version_dir/" 2>/dev/null || true
|
||||
shopt -u dotglob
|
||||
# run component installer within version dir
|
||||
if [[ -f "$version_dir/install.sh" ]]; then
|
||||
chmod +x "$version_dir/install.sh" 2>/dev/null || true
|
||||
# 传递运行时开关：容器内缺省启用 AUTO_START_DCGM=1、禁用 Profiling（可通过环境变量覆盖）
|
||||
# 注意：不能用 `VAR=.. VAR2=.. (cmd)` 前缀到子 shell；bash 不允许 env 赋值直接修饰 `(` 复合命令。
|
||||
# 因此改为在子 subshell 中 export 后再执行。
|
||||
(
|
||||
export AUTO_START_DCGM="${AUTO_START_DCGM:-1}"
|
||||
export DCGM_EXPORTER_DISABLE_PROFILING="${DCGM_EXPORTER_DISABLE_PROFILING:-1}"
|
||||
export DCGM_EXPORTER_LISTEN="${DCGM_EXPORTER_LISTEN:-:9400}"
|
||||
cd "$version_dir" && ./install.sh "$version_dir"
|
||||
)
|
||||
echo "$ver" > "$target_root/LATEST_VERSION" 2>/dev/null || true
|
||||
ln -sfn "$version_dir" "$target_root/current" 2>/dev/null || true
|
||||
if [[ -L "$target_root/current" && -d "$target_root/current" ]]; then
|
||||
installed_ok=1
|
||||
echo "[BOOT] local bundle install OK: version=$ver"
|
||||
else
|
||||
echo "[BOOT][WARN] current symlink not present after install; will rely on healthcheck to confirm"
|
||||
fi
|
||||
else
|
||||
echo "[BOOT][WARN] install.sh missing under $version_dir; fallback to FTP"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# 3) fallback: use FTP setup if not installed
|
||||
if [[ ! -L "$INSTALL_DIR/current" && "$installed_ok" -eq 0 ]]; then
|
||||
echo "[BOOT] fallback to FTP setup"
|
||||
if [[ -z "${FTPIP:-}" || -z "${FTP_USER:-}" || -z "${FTP_PASSWORD:-}" ]]; then
|
||||
echo "[BOOT][ERROR] FTP variables not set (FTPIP/FTP_USER/FTP_PASSWORD)" >&2
|
||||
exit 1
|
||||
fi
|
||||
curl -u "$FTP_USER:$FTP_PASSWORD" -fsSL "ftp://$FTPIP:21/setup.sh" -o /tmp/setup.sh
|
||||
chmod +x /tmp/setup.sh
|
||||
/tmp/setup.sh --server "$FTPIP" --user "$FTP_USER" --password "$FTP_PASSWORD" --port 21
|
||||
fi
|
||||
fi
|
||||
|
||||
# 4) ensure agent is running; start if needed (inherits env: MASTER_ENDPOINT/AGENT_*)
|
||||
if ! pgrep -x argus-agent >/dev/null 2>&1; then
|
||||
echo "[BOOT] starting argus-agent (not detected)"
|
||||
setsid /usr/local/bin/argus-agent >/var/log/argus-agent.log 2>&1 < /dev/null &
|
||||
fi
|
||||
|
||||
# 5) 若 dcgm-exporter 未监听（可能因 Profiling 崩溃），尝试无 Profiling 清单回退启动
|
||||
if ! ss -tlnp 2>/dev/null | grep -q ":9400 "; then
|
||||
echo "[BOOT] dcgm-exporter not listening; trying no-prof fallback"
|
||||
pgrep -f nv-hostengine >/dev/null || (nohup nv-hostengine >/var/log/nv-hostengine.log 2>&1 & sleep 2)
|
||||
cfg_dir="/etc/dcgm-exporter"; default_cfg="$cfg_dir/default-counters.csv"; no_prof_cfg="$cfg_dir/no-prof.csv"
|
||||
if [[ -f "$default_cfg" ]]; then
|
||||
grep -v 'DCGM_FI_PROF_' "$default_cfg" > "$no_prof_cfg" || true
|
||||
pkill -f dcgm-exporter >/dev/null 2>&1 || true
|
||||
nohup /usr/local/bin/dcgm-exporter --address="${DCGM_EXPORTER_LISTEN:-:9400}" --collectors "$no_prof_cfg" >/var/log/dcgm-exporter.log 2>&1 &
|
||||
fi
|
||||
fi
|
||||
|
||||
# 6) post-install selfcheck (best-effort) and wait for node.json
|
||||
for i in {1..30}; do
|
||||
if compgen -G "$INSTALL_DIR/versions/*/check_health.sh" > /dev/null; then
|
||||
bash "$INSTALL_DIR"/versions/*/check_health.sh || true
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
host="$(hostname)"
|
||||
state_dir="/private/argus/agent/${host}"
|
||||
mkdir -p "$state_dir" 2>/dev/null || true
|
||||
for i in {1..60}; do
|
||||
if [[ -s "$state_dir/node.json" ]]; then
|
||||
echo "[BOOT] node state present: $state_dir/node.json"
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# 7) spawn health watcher (best-effort, non-blocking)
|
||||
ver_dir=""
|
||||
if [[ -L "$INSTALL_DIR/current" ]]; then
|
||||
ver_dir="$(readlink -f "$INSTALL_DIR/current" 2>/dev/null || true)"
|
||||
fi
|
||||
if [[ -z "$ver_dir" ]]; then
|
||||
ver_dir="$(ls -d "$INSTALL_DIR"/versions/* 2>/dev/null | sort -V | tail -n1 || true)"
|
||||
fi
|
||||
|
||||
if command -v /usr/local/bin/health-watcher.sh >/dev/null 2>&1; then
|
||||
echo "[BOOT] starting health watcher for $ver_dir"
|
||||
setsid /usr/local/bin/health-watcher.sh "${ver_dir:-}" >/var/log/health-watcher.log 2>&1 < /dev/null || true &
|
||||
else
|
||||
echo "[BOOT][WARN] health-watcher.sh not found; skip health watcher"
|
||||
fi
|
||||
|
||||
echo "[BOOT] ready; entering sleep"
|
||||
exec sleep infinity
|
||||
Reference in New Issue
Block a user