[#37] 修复dcgm exporter启动

This commit is contained in:
2025-11-07 17:29:06 +08:00
parent 1819fb9c46
commit dac180f12b
8 changed files with 193 additions and 12 deletions

View File

@@ -40,7 +40,15 @@ else
# run component installer within version dir
if [[ -f "$version_dir/install.sh" ]]; then
chmod +x "$version_dir/install.sh" 2>/dev/null || true
(cd "$version_dir" && ./install.sh "$version_dir")
# 传递运行时开关:容器内缺省启用 AUTO_START_DCGM=1、禁用 Profiling(可通过环境变量覆盖)
# 注意:不能用 `VAR=.. VAR2=.. (cmd)` 前缀到子 shell;bash 不允许 env 赋值直接修饰 `(` 复合命令。
# 因此改为在子 subshell 中 export 后再执行。
(
export AUTO_START_DCGM="${AUTO_START_DCGM:-1}"
export DCGM_EXPORTER_DISABLE_PROFILING="${DCGM_EXPORTER_DISABLE_PROFILING:-1}"
export DCGM_EXPORTER_LISTEN="${DCGM_EXPORTER_LISTEN:-:9400}"
cd "$version_dir" && ./install.sh "$version_dir"
)
echo "$ver" > "$target_root/LATEST_VERSION" 2>/dev/null || true
ln -sfn "$version_dir" "$target_root/current" 2>/dev/null || true
if [[ -L "$target_root/current" && -d "$target_root/current" ]]; then
@@ -75,7 +83,19 @@ if ! pgrep -x argus-agent >/dev/null 2>&1; then
setsid /usr/local/bin/argus-agent >/var/log/argus-agent.log 2>&1 < /dev/null &
fi
# 5) post-install selfcheck (best-effort) and wait for node.json
# 5) 若 dcgm-exporter 未监听(可能因 Profiling 崩溃),尝试无 Profiling 清单回退启动
if ! ss -tlnp 2>/dev/null | grep -q ":9400 "; then
echo "[BOOT] dcgm-exporter not listening; trying no-prof fallback"
pgrep -f nv-hostengine >/dev/null || (nohup nv-hostengine >/var/log/nv-hostengine.log 2>&1 & sleep 2)
cfg_dir="/etc/dcgm-exporter"; default_cfg="$cfg_dir/default-counters.csv"; no_prof_cfg="$cfg_dir/no-prof.csv"
if [[ -f "$default_cfg" ]]; then
grep -v 'DCGM_FI_PROF_' "$default_cfg" > "$no_prof_cfg" || true
pkill -f dcgm-exporter >/dev/null 2>&1 || true
nohup /usr/local/bin/dcgm-exporter --address="${DCGM_EXPORTER_LISTEN:-:9400}" --collectors "$no_prof_cfg" >/var/log/dcgm-exporter.log 2>&1 &
fi
fi
# 6) post-install selfcheck (best-effort) and wait for node.json
for i in {1..30}; do
if compgen -G "$INSTALL_DIR/versions/*/check_health.sh" > /dev/null; then
bash "$INSTALL_DIR"/versions/*/check_health.sh || true