[#37] 修复dcgm exporter启动
This commit is contained in:
@@ -40,7 +40,15 @@ else
|
||||
# run component installer within version dir
|
||||
if [[ -f "$version_dir/install.sh" ]]; then
|
||||
chmod +x "$version_dir/install.sh" 2>/dev/null || true
|
||||
(cd "$version_dir" && ./install.sh "$version_dir")
|
||||
# 传递运行时开关:容器内缺省启用 AUTO_START_DCGM=1、禁用 Profiling(可通过环境变量覆盖)
|
||||
# 注意:不能用 `VAR=.. VAR2=.. (cmd)` 前缀到子 shell;bash 不允许 env 赋值直接修饰 `(` 复合命令。
|
||||
# 因此改为在子 subshell 中 export 后再执行。
|
||||
(
|
||||
export AUTO_START_DCGM="${AUTO_START_DCGM:-1}"
|
||||
export DCGM_EXPORTER_DISABLE_PROFILING="${DCGM_EXPORTER_DISABLE_PROFILING:-1}"
|
||||
export DCGM_EXPORTER_LISTEN="${DCGM_EXPORTER_LISTEN:-:9400}"
|
||||
cd "$version_dir" && ./install.sh "$version_dir"
|
||||
)
|
||||
echo "$ver" > "$target_root/LATEST_VERSION" 2>/dev/null || true
|
||||
ln -sfn "$version_dir" "$target_root/current" 2>/dev/null || true
|
||||
if [[ -L "$target_root/current" && -d "$target_root/current" ]]; then
|
||||
@@ -75,7 +83,19 @@ if ! pgrep -x argus-agent >/dev/null 2>&1; then
|
||||
setsid /usr/local/bin/argus-agent >/var/log/argus-agent.log 2>&1 < /dev/null &
|
||||
fi
|
||||
|
||||
# 5) post-install selfcheck (best-effort) and wait for node.json
|
||||
# 5) 若 dcgm-exporter 未监听(可能因 Profiling 崩溃),尝试无 Profiling 清单回退启动
|
||||
if ! ss -tlnp 2>/dev/null | grep -q ":9400 "; then
|
||||
echo "[BOOT] dcgm-exporter not listening; trying no-prof fallback"
|
||||
pgrep -f nv-hostengine >/dev/null || (nohup nv-hostengine >/var/log/nv-hostengine.log 2>&1 & sleep 2)
|
||||
cfg_dir="/etc/dcgm-exporter"; default_cfg="$cfg_dir/default-counters.csv"; no_prof_cfg="$cfg_dir/no-prof.csv"
|
||||
if [[ -f "$default_cfg" ]]; then
|
||||
grep -v 'DCGM_FI_PROF_' "$default_cfg" > "$no_prof_cfg" || true
|
||||
pkill -f dcgm-exporter >/dev/null 2>&1 || true
|
||||
nohup /usr/local/bin/dcgm-exporter --address="${DCGM_EXPORTER_LISTEN:-:9400}" --collectors "$no_prof_cfg" >/var/log/dcgm-exporter.log 2>&1 &
|
||||
fi
|
||||
fi
|
||||
|
||||
# 6) post-install selfcheck (best-effort) and wait for node.json
|
||||
for i in {1..30}; do
|
||||
if compgen -G "$INSTALL_DIR/versions/*/check_health.sh" > /dev/null; then
|
||||
bash "$INSTALL_DIR"/versions/*/check_health.sh || true
|
||||
|
||||
Reference in New Issue
Block a user