[#39] 增加新docker compose部署方式，验证通过gpu部署；剩余fluent-bit logs目录权限问题 以及 gpu bundle打包优化问题
This commit is contained in:
81
deployment_new/templates/client_gpu/scripts/config.sh
Normal file
81
deployment_new/templates/client_gpu/scripts/config.sh
Normal file
@@ -0,0 +1,81 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PKG_ROOT="$ROOT_DIR"
|
||||
ENV_EX="$PKG_ROOT/compose/.env.example"
|
||||
ENV_OUT="$PKG_ROOT/compose/.env"
|
||||
|
||||
info(){ echo -e "\033[34m[CONFIG-GPU]\033[0m $*"; }
|
||||
err(){ echo -e "\033[31m[ERROR]\033[0m $*" >&2; }
|
||||
require(){ local ok=1; for c in "$@"; do command -v "$c" >/dev/null 2>&1 || { err "缺少依赖: $c"; ok=0; }; done; [[ $ok -eq 1 ]]; }
|
||||
require docker curl jq awk sed tar gzip nvidia-smi
|
||||
|
||||
# 磁盘空间检查（MB）
|
||||
check_disk(){ local p="$1"; local need=10240; local free
|
||||
free=$(df -Pm "$p" | awk 'NR==2{print $4+0}')
|
||||
if [[ -z "$free" || "$free" -lt "$need" ]]; then err "磁盘空间不足: $p 剩余 ${free:-0}MB (<${need}MB)"; return 1; fi
|
||||
}
|
||||
check_disk "$PKG_ROOT"; check_disk "/var/lib/docker" || true
|
||||
|
||||
# 导入 cluster-info.env（默认取当前包根，也可用 CLUSTER_INFO 指定路径）
|
||||
CI_IN="${CLUSTER_INFO:-$PKG_ROOT/cluster-info.env}"
|
||||
info "读取 cluster-info.env: $CI_IN"
|
||||
[[ -f "$CI_IN" ]] || { err "找不到 cluster-info.env（默认当前包根，或设置环境变量 CLUSTER_INFO 指定绝对路径）"; exit 1; }
|
||||
set -a; source "$CI_IN"; set +a
|
||||
[[ -n "${SWARM_MANAGER_ADDR:-}" && -n "${SWARM_JOIN_TOKEN_WORKER:-}" ]] || { err "cluster-info.env 缺少 SWARM 信息（SWARM_MANAGER_ADDR/SWARM_JOIN_TOKEN_WORKER）"; exit 1; }
|
||||
|
||||
# 加入 Swarm（幂等）
|
||||
info "加入 Swarm（幂等）：$SWARM_MANAGER_ADDR"
|
||||
docker swarm join --token "$SWARM_JOIN_TOKEN_WORKER" "$SWARM_MANAGER_ADDR":2377 >/dev/null 2>&1 || true
|
||||
|
||||
# 导入 busybox 并做 overlay 预热与连通性（总是执行）
|
||||
NET_NAME="${ARGUS_OVERLAY_NET:-argus-sys-net}"
|
||||
# 准备 busybox
|
||||
if ! docker image inspect busybox:latest >/dev/null 2>&1; then
|
||||
if [[ -f "$PKG_ROOT/images/busybox.tar" ]]; then
|
||||
info "加载 busybox.tar 以预热 overlay"
|
||||
docker load -i "$PKG_ROOT/images/busybox.tar" >/dev/null
|
||||
else
|
||||
err "缺少 busybox 镜像（包内 images/busybox.tar 或本地 busybox:latest），无法预热 overlay $NET_NAME"; exit 1
|
||||
fi
|
||||
fi
|
||||
# 预热容器（worker 侧加入 overlay 以便本地可见）
|
||||
docker rm -f argus-net-warmup >/dev/null 2>&1 || true
|
||||
info "启动 warmup 容器加入 overlay: $NET_NAME"
|
||||
docker run -d --rm --name argus-net-warmup --network "$NET_NAME" ${BINDIP:+--dns "$BINDIP"} busybox:latest sleep 600 >/dev/null 2>&1 || true
|
||||
for i in {1..60}; do docker network inspect "$NET_NAME" >/dev/null 2>&1 && { info "overlay 可见 (t=${i}s)"; break; }; sleep 1; done
|
||||
docker network inspect "$NET_NAME" >/dev/null 2>&1 || { err "预热后仍未看到 overlay: $NET_NAME；请确认 manager 已创建并网络可达"; exit 1; }
|
||||
|
||||
# 从 warmup 容器内测试连通性（必须能 ping 通 BINDIP 与 FTPIP）
|
||||
ping_ok(){ docker exec argus-net-warmup sh -lc "ping -c 1 -W 2 $1 >/dev/null 2>&1"; }
|
||||
if [[ -n "${BINDIP:-}" ]]; then
|
||||
ping_ok "$BINDIP" || { err "容器内无法 ping 通 BINDIP=$BINDIP；请检查 overlay 与 Bind9 容器状态"; exit 1; }
|
||||
fi
|
||||
if [[ -n "${FTPIP:-}" ]]; then
|
||||
ping_ok "$FTPIP" || { err "容器内无法 ping 通 FTPIP=$FTPIP；请检查 overlay 与 FTP 容器状态"; exit 1; }
|
||||
fi
|
||||
|
||||
# 生成/更新 .env（保留人工填写项，不覆盖已有键）
|
||||
if [[ ! -f "$ENV_OUT" ]]; then
|
||||
cp "$ENV_EX" "$ENV_OUT"
|
||||
fi
|
||||
|
||||
set_kv(){ local k="$1" v="$2"; if grep -q "^${k}=" "$ENV_OUT"; then sed -i -E "s#^${k}=.*#${k}=${v}#" "$ENV_OUT"; else echo "${k}=${v}" >> "$ENV_OUT"; fi }
|
||||
|
||||
set_kv BINDIP "${BINDIP:-}"
|
||||
set_kv FTPIP "${FTPIP:-}"
|
||||
set_kv SWARM_MANAGER_ADDR "${SWARM_MANAGER_ADDR:-}"
|
||||
set_kv SWARM_JOIN_TOKEN_WORKER "${SWARM_JOIN_TOKEN_WORKER:-}"
|
||||
set_kv SWARM_JOIN_TOKEN_MANAGER "${SWARM_JOIN_TOKEN_MANAGER:-}"
|
||||
|
||||
REQ_VARS=(AGENT_ENV AGENT_USER AGENT_INSTANCE GPU_NODE_HOSTNAME)
|
||||
missing=()
|
||||
for v in "${REQ_VARS[@]}"; do
|
||||
val=$(grep -E "^$v=" "$ENV_OUT" | head -1 | cut -d= -f2-)
|
||||
if [[ -z "$val" ]]; then missing+=("$v"); fi
|
||||
done
|
||||
if [[ ${#missing[@]} -gt 0 ]]; then
|
||||
err "以下变量必须在 compose/.env 中填写：${missing[*]}（已保留你现有的内容，不会被覆盖）"; exit 1; fi
|
||||
|
||||
info "已生成 compose/.env；可执行 scripts/install.sh"
|
||||
57
deployment_new/templates/client_gpu/scripts/install.sh
Normal file
57
deployment_new/templates/client_gpu/scripts/install.sh
Normal file
@@ -0,0 +1,57 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PKG_ROOT="$ROOT_DIR"
|
||||
ENV_FILE="$PKG_ROOT/compose/.env"
|
||||
COMPOSE_FILE="$PKG_ROOT/compose/docker-compose.yml"
|
||||
|
||||
info(){ echo -e "\033[34m[INSTALL-GPU]\033[0m $*"; }
|
||||
err(){ echo -e "\033[31m[ERROR]\033[0m $*" >&2; }
|
||||
require(){ local ok=1; for c in "$@"; do command -v "$c" >/dev/null 2>&1 || { err "缺少依赖: $c"; ok=0; }; done; [[ $ok -eq 1 ]]; }
|
||||
require docker docker compose nvidia-smi
|
||||
|
||||
[[ -f "$ENV_FILE" ]] || { err "缺少 compose/.env，请先运行 scripts/config.sh"; exit 1; }
|
||||
info "使用环境文件: $ENV_FILE"
|
||||
|
||||
# 预热 overlay（当 config 执行很久之前或容器已被清理时，warmup 可能不存在）
|
||||
set -a; source "$ENV_FILE"; set +a
|
||||
NET_NAME="${ARGUS_OVERLAY_NET:-argus-sys-net}"
|
||||
info "检查 overlay 网络可见性: $NET_NAME"
|
||||
if ! docker network inspect "$NET_NAME" >/dev/null 2>&1; then
|
||||
# 如 Overlay 不可见，尝试用 busybox 预热
|
||||
if ! docker image inspect busybox:latest >/dev/null 2>&1; then
|
||||
if [[ -f "$PKG_ROOT/images/busybox.tar" ]]; then docker load -i "$PKG_ROOT/images/busybox.tar"; else err "缺少 busybox 镜像（images/busybox.tar 或本地 busybox:latest）"; exit 1; fi
|
||||
fi
|
||||
docker rm -f argus-net-warmup >/dev/null 2>&1 || true
|
||||
docker run -d --rm --name argus-net-warmup --network "$NET_NAME" ${BINDIP:+--dns "$BINDIP"} busybox:latest sleep 600 >/dev/null 2>&1 || true
|
||||
for i in {1..60}; do docker network inspect "$NET_NAME" >/dev/null 2>&1 && break; sleep 1; done
|
||||
docker network inspect "$NET_NAME" >/dev/null 2>&1 || { err "预热后仍未看到 overlay: $NET_NAME；请确认 manager 已创建并网络可达"; exit 1; }
|
||||
info "overlay 已可见（warmup=argus-net-warmup）"
|
||||
fi
|
||||
|
||||
# 容器内连通性检查：BINDIP 与 FTPIP 可达
|
||||
ping_ok(){ docker exec argus-net-warmup sh -lc "ping -c 1 -W 2 $1 >/dev/null 2>&1"; }
|
||||
if [[ -n "${BINDIP:-}" ]]; then
|
||||
if ping_ok "$BINDIP"; then info "warmup 内可达 BINDIP=$BINDIP"; else err "容器内无法 ping 通 BINDIP=$BINDIP"; exit 1; fi
|
||||
fi
|
||||
if [[ -n "${FTPIP:-}" ]]; then
|
||||
if ping_ok "$FTPIP"; then info "warmup 内可达 FTPIP=$FTPIP"; else err "容器内无法 ping 通 FTPIP=$FTPIP"; exit 1; fi
|
||||
fi
|
||||
|
||||
# 导入 GPU bundle 镜像
|
||||
IMG_TGZ=$(ls -1 "$PKG_ROOT"/images/argus-sys-metric-test-node-bundle-gpu-*.tar.gz 2>/dev/null | head -1 || true)
|
||||
[[ -n "$IMG_TGZ" ]] || { err "找不到 GPU bundle 镜像 tar.gz"; exit 1; }
|
||||
info "导入 GPU bundle 镜像: $(basename "$IMG_TGZ")"
|
||||
tmp=$(mktemp); gunzip -c "$IMG_TGZ" > "$tmp"; docker load -i "$tmp" >/dev/null; rm -f "$tmp"
|
||||
|
||||
# 确保日志目录存在（宿主侧，用于映射 /logs/infer 与 /logs/train）
|
||||
mkdir -p "$PKG_ROOT/logs/infer" "$PKG_ROOT/logs/train"
|
||||
info "日志目录已准备: logs/infer logs/train"
|
||||
|
||||
# 启动 compose 并跟踪日志
|
||||
info "启动 GPU 节点 (docker compose up -d)"
|
||||
docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" up -d
|
||||
docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" ps
|
||||
info "跟踪节点容器日志（按 Ctrl+C 退出）"
|
||||
docker logs -f argus-metric-gpu-node-swarm || true
|
||||
25
deployment_new/templates/client_gpu/scripts/uninstall.sh
Normal file
25
deployment_new/templates/client_gpu/scripts/uninstall.sh
Normal file
@@ -0,0 +1,25 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PKG_ROOT="$ROOT_DIR"
|
||||
ENV_FILE="$PKG_ROOT/compose/.env"
|
||||
COMPOSE_FILE="$PKG_ROOT/compose/docker-compose.yml"
|
||||
|
||||
info(){ echo -e "\033[34m[UNINSTALL-GPU]\033[0m $*"; }
|
||||
|
||||
if [[ -f "$ENV_FILE" ]]; then
|
||||
info "stopping compose project"
|
||||
docker compose --env-file "$ENV_FILE" -f "$COMPOSE_FILE" down --remove-orphans || true
|
||||
else
|
||||
info "compose/.env not found; attempting to remove container by name"
|
||||
fi
|
||||
|
||||
# remove warmup container if still running
|
||||
docker rm -f argus-net-warmup >/dev/null 2>&1 || true
|
||||
|
||||
# remove node container if present
|
||||
docker rm -f argus-metric-gpu-node-swarm >/dev/null 2>&1 || true
|
||||
|
||||
info "uninstall completed"
|
||||
|
||||
Reference in New Issue
Block a user