[#49] swarm test重启测试通过

This commit is contained in:
2025-11-19 17:26:26 +08:00
parent 1d38304936
commit d4e0dc1511
19 changed files with 697 additions and 116 deletions

View File

@@ -19,10 +19,6 @@ services:
# Fluent Bit / 日志上报目标(固定域名)
- ES_HOST=es.log.argus.com
- ES_PORT=9200
- FTPIP=${FTPIP}
- BINDIP=${BINDIP}
- FTP_USER=${FTP_USER:-ftpuser}
- FTP_PASSWORD=${FTP_PASSWORD:-NASPlab1234!}
- ARGUS_BUILD_UID=${ARGUS_BUILD_UID:-2133}
- ARGUS_BUILD_GID=${ARGUS_BUILD_GID:-2015}
- AGENT_ENV=${AGENT_ENV}
@@ -31,9 +27,10 @@ services:
- NVIDIA_VISIBLE_DEVICES=all
- NVIDIA_DRIVER_CAPABILITIES=compute,utility
- GPU_MODE=gpu
dns:
- ${BINDIP}
networks: [argus-sys-net]
networks:
argus-sys-net:
aliases:
- ${AGENT_INSTANCE}.node.argus.com
volumes:
- ../private/argus/agent:/private/argus/agent
- ../logs/infer:/logs/infer

View File

@@ -12,7 +12,7 @@
su - argus -c 'id; docker ps >/dev/null && echo OK || echo NO_DOCKER_PERMISSION'
```
后续解压与执行(config/install/uninstall)均使用 `argus` 账户进行。
- 从 Server 安装方拿到 `cluster-info.env`(包含 `SWARM_MANAGER_ADDR/BINDIP/FTPIP/SWARM_JOIN_TOKEN_*`)。
- 从 Server 安装方拿到 `cluster-info.env`(包含 `SWARM_MANAGER_ADDR/SWARM_JOIN_TOKEN_*`;compose 架构下 BINDIP/FTPIP 不再使用)。
## 二、解包
- `tar -xzf client_gpu_YYYYMMDD.tar.gz`
@@ -28,13 +28,13 @@ cp /path/to/cluster-info.env ./ # 或 export CLUSTER_INFO=/abs/path/cluster-in
脚本做了什么:
- 读取 `cluster-info.env` 并 `docker swarm join`(幂等);
- 自动用 busybox 预热 external overlay `argus-sys-net`,等待最多 60s 直到本机可见;
- 生成/更新 `compose/.env`:填入 `BINDIP/FTPIP/SWARM_*`,并“保留你已填写的 AGENT_* 与 GPU_NODE_HOSTNAME”(不会覆盖)。
- 生成/更新 `compose/.env`:填入 `SWARM_*`,并“保留你已填写的 AGENT_* 与 GPU_NODE_HOSTNAME”(不会覆盖)。
看到什么才算成功:
- 终端输出类似:`已预热 overlay=argus-sys-net 并生成 compose/.env;可执行 scripts/install.sh`;
- `compose/.env` 至少包含:
- `AGENT_ENV/AGENT_USER/AGENT_INSTANCE/GPU_NODE_HOSTNAME`(需要你提前填写);
- `BINDIP/FTPIP/SWARM_MANAGER_ADDR/SWARM_JOIN_TOKEN_*`;
- `SWARM_MANAGER_ADDR/SWARM_JOIN_TOKEN_*`;
- `NODE_GPU_BUNDLE_IMAGE_TAG=...:YYYYMMDD`。
### 日志映射(重要)

View File

@@ -50,18 +50,16 @@ fi
# 预热容器(worker 侧加入 overlay 以便本地可见)
docker rm -f argus-net-warmup >/dev/null 2>&1 || true
info "启动 warmup 容器加入 overlay: $NET_NAME"
docker run -d --rm --name argus-net-warmup --network "$NET_NAME" ${BINDIP:+--dns "$BINDIP"} busybox:latest sleep 600 >/dev/null 2>&1 || true
docker run -d --rm --name argus-net-warmup --network "$NET_NAME" busybox:latest sleep 600 >/dev/null 2>&1 || true
for i in {1..60}; do docker network inspect "$NET_NAME" >/dev/null 2>&1 && { info "overlay 可见 (t=${i}s)"; break; }; sleep 1; done
docker network inspect "$NET_NAME" >/dev/null 2>&1 || { err "预热后仍未看到 overlay: $NET_NAME;请确认 manager 已创建并网络可达"; exit 1; }
# 从 warmup 容器内测试连通性(必须能 ping 通 BINDIP 与 FTPIP)
ping_ok(){ docker exec argus-net-warmup sh -lc "ping -c 1 -W 2 $1 >/dev/null 2>&1"; }
if [[ -n "${BINDIP:-}" ]]; then
ping_ok "$BINDIP" || { err "容器内无法 ping 通 BINDIP=$BINDIP;请检查 overlay 与 Bind9 容器状态"; exit 1; }
fi
if [[ -n "${FTPIP:-}" ]]; then
ping_ok "$FTPIP" || { err "容器内无法 ping 通 FTPIP=$FTPIP;请检查 overlay 与 FTP 容器状态"; exit 1; }
# 通过 warmup 容器测试实际数据通路(alias → master)
if ! docker exec argus-net-warmup sh -lc "ping -c 1 -W 2 master.argus.com >/dev/null 2>&1"; then
err "warmup 容器内无法通过别名访问 master.argus.com;请确认 server compose 已启动并加入 overlay $NET_NAME"
exit 1
fi
info "warmup 容器内可达 master.argus.com(Docker DNS + alias 正常)"
# 生成/更新 .env(保留人工填写项,不覆盖已有键)
if [[ ! -f "$ENV_OUT" ]]; then
@@ -70,8 +68,6 @@ fi
set_kv(){ local k="$1" v="$2"; if grep -q "^${k}=" "$ENV_OUT"; then sed -i -E "s#^${k}=.*#${k}=${v}#" "$ENV_OUT"; else echo "${k}=${v}" >> "$ENV_OUT"; fi }
set_kv BINDIP "${BINDIP:-}"
set_kv FTPIP "${FTPIP:-}"
set_kv SWARM_MANAGER_ADDR "${SWARM_MANAGER_ADDR:-}"
set_kv SWARM_JOIN_TOKEN_WORKER "${SWARM_JOIN_TOKEN_WORKER:-}"
set_kv SWARM_JOIN_TOKEN_MANAGER "${SWARM_JOIN_TOKEN_MANAGER:-}"

View File

@@ -26,24 +26,24 @@ set -a; source "$ENV_FILE"; set +a
NET_NAME="${ARGUS_OVERLAY_NET:-argus-sys-net}"
info "检查 overlay 网络可见性: $NET_NAME"
if ! docker network inspect "$NET_NAME" >/dev/null 2>&1; then
# 如 Overlay 不可见,尝试用 busybox 预热
# 如 Overlay 不可见,尝试用 busybox 预热(仅为确保 worker 节点已加入 overlay)
if ! docker image inspect busybox:latest >/dev/null 2>&1; then
if [[ -f "$PKG_ROOT/images/busybox.tar" ]]; then docker load -i "$PKG_ROOT/images/busybox.tar"; else err "缺少 busybox 镜像(images/busybox.tar 或本地 busybox:latest)"; exit 1; fi
fi
docker rm -f argus-net-warmup >/dev/null 2>&1 || true
docker run -d --rm --name argus-net-warmup --network "$NET_NAME" ${BINDIP:+--dns "$BINDIP"} busybox:latest sleep 600 >/dev/null 2>&1 || true
docker run -d --rm --name argus-net-warmup --network "$NET_NAME" busybox:latest sleep 600 >/dev/null 2>&1 || true
for i in {1..60}; do docker network inspect "$NET_NAME" >/dev/null 2>&1 && break; sleep 1; done
docker network inspect "$NET_NAME" >/dev/null 2>&1 || { err "预热后仍未看到 overlay: $NET_NAME;请确认 manager 已创建并网络可达"; exit 1; }
info "overlay 已可见(warmup=argus-net-warmup)"
fi
# 容器内连通性检查:BINDIP 与 FTPIP 可达
ping_ok(){ docker exec argus-net-warmup sh -lc "ping -c 1 -W 2 $1 >/dev/null 2>&1"; }
if [[ -n "${BINDIP:-}" ]]; then
if ping_ok "$BINDIP"; then info "warmup 内可达 BINDIP=$BINDIP"; else err "容器内无法 ping 通 BINDIP=$BINDIP"; exit 1; fi
fi
if [[ -n "${FTPIP:-}" ]]; then
if ping_ok "$FTPIP"; then info "warmup 内可达 FTPIP=$FTPIP"; else err "容器内无法 ping 通 FTPIP=$FTPIP"; exit 1; fi
# 若本函数内重新创建了 warmup 容器,同样测试一次 alias 数据通路
if docker ps --format '{{.Names}}' | grep -q '^argus-net-warmup$'; then
if ! docker exec argus-net-warmup sh -lc "ping -c 1 -W 2 master.argus.com >/dev/null 2>&1"; then
err "GPU install 阶段:warmup 容器内无法通过别名访问 master.argus.com;请检查 overlay $NET_NAME 与 server 状态"
exit 1
fi
info "GPU install 阶段:warmup 容器内可达 master.argus.com"
fi
# 导入 GPU bundle 镜像

View File

@@ -5,18 +5,9 @@ networks:
external: true
services:
bind:
image: ${BIND_IMAGE_TAG:-argus-bind9:${PKG_VERSION}}
container_name: argus-bind-sys
networks: [argus-sys-net]
volumes:
- ../private:/private
restart: unless-stopped
master:
image: ${MASTER_IMAGE_TAG:-argus-master:${PKG_VERSION}}
container_name: argus-master-sys
depends_on: [bind]
environment:
- OFFLINE_THRESHOLD_SECONDS=6
- ONLINE_THRESHOLD_SECONDS=2
@@ -29,7 +20,10 @@ services:
- ../private/argus/master:/private/argus/master
- ../private/argus/metric/prometheus:/private/argus/metric/prometheus
- ../private/argus/etc:/private/argus/etc
networks: [argus-sys-net]
networks:
argus-sys-net:
aliases:
- master.argus.com
restart: unless-stopped
es:
@@ -47,7 +41,10 @@ services:
ports:
- "${ES_HTTP_PORT:-9200}:9200"
restart: unless-stopped
networks: [argus-sys-net]
networks:
argus-sys-net:
aliases:
- es.log.argus.com
kibana:
image: ${KIBANA_IMAGE_TAG:-argus-kibana:${PKG_VERSION}}
@@ -63,27 +60,10 @@ services:
ports:
- "${KIBANA_PORT:-5601}:5601"
restart: unless-stopped
networks: [argus-sys-net]
ftp:
image: ${FTP_IMAGE_TAG:-argus-metric-ftp:${PKG_VERSION}}
container_name: argus-ftp
restart: unless-stopped
environment:
- TZ=Asia/Shanghai
- FTP_BASE_PATH=/private/argus/ftp
- FTP_PASSWORD=${FTP_PASSWORD:-NASPlab1234!}
- DOMAIN=${FTP_DOMAIN:-ftp.metric.argus.com}
- ARGUS_BUILD_UID=${ARGUS_BUILD_UID:-2133}
- ARGUS_BUILD_GID=${ARGUS_BUILD_GID:-2015}
ports:
- "${FTP_PORT:-21}:21"
- "${FTP_DATA_PORT:-20}:20"
- "${FTP_PASSIVE_HOST_RANGE:-21100-21110}:21100-21110"
volumes:
- ../private/argus/metric/ftp:/private/argus/ftp
- ../private/argus/etc:/private/argus/etc
networks: [argus-sys-net]
networks:
argus-sys-net:
aliases:
- kibana.log.argus.com
prometheus:
image: ${PROM_IMAGE_TAG:-argus-metric-prometheus:${PKG_VERSION}}
@@ -99,7 +79,10 @@ services:
volumes:
- ../private/argus/metric/prometheus:/private/argus/metric/prometheus
- ../private/argus/etc:/private/argus/etc
networks: [argus-sys-net]
networks:
argus-sys-net:
aliases:
- prom.metric.argus.com
grafana:
image: ${GRAFANA_IMAGE_TAG:-argus-metric-grafana:${PKG_VERSION}}
@@ -122,7 +105,10 @@ services:
- ../private/argus/metric/grafana:/private/argus/metric/grafana
- ../private/argus/etc:/private/argus/etc
depends_on: [prometheus]
networks: [argus-sys-net]
networks:
argus-sys-net:
aliases:
- grafana.metric.argus.com
alertmanager:
image: ${ALERT_IMAGE_TAG:-argus-alertmanager:${PKG_VERSION}}
@@ -133,7 +119,10 @@ services:
volumes:
- ../private/argus/etc:/private/argus/etc
- ../private/argus/alert/alertmanager:/private/argus/alert/alertmanager
networks: [argus-sys-net]
networks:
argus-sys-net:
aliases:
- alertmanager.alert.argus.com
ports:
- "${ALERTMANAGER_PORT:-9093}:9093"
restart: unless-stopped
@@ -151,19 +140,25 @@ services:
- EXTERNAL_KIBANA_PORT=${WEB_PROXY_PORT_8083:-8083}
volumes:
- ../private/argus/etc:/private/argus/etc
networks: [argus-sys-net]
networks:
argus-sys-net:
aliases:
- web.argus.com
restart: unless-stopped
web-proxy:
image: ${WEB_PROXY_IMAGE_TAG:-argus-web-proxy:${PKG_VERSION}}
container_name: argus-web-proxy
depends_on: [bind, master, grafana, prometheus, kibana, alertmanager]
depends_on: [master, grafana, prometheus, kibana, alertmanager]
environment:
- ARGUS_BUILD_UID=${ARGUS_BUILD_UID:-2133}
- ARGUS_BUILD_GID=${ARGUS_BUILD_GID:-2015}
volumes:
- ../private/argus/etc:/private/argus/etc
networks: [argus-sys-net]
networks:
argus-sys-net:
aliases:
- proxy.argus.com
ports:
- "${WEB_PROXY_PORT_8080:-8080}:8080"
- "${WEB_PROXY_PORT_8081:-8081}:8081"
@@ -172,4 +167,3 @@ services:
- "${WEB_PROXY_PORT_8084:-8084}:8084"
- "${WEB_PROXY_PORT_8085:-8085}:8085"
restart: unless-stopped

View File

@@ -71,7 +71,7 @@ export SWARM_MANAGER_ADDR=<本机管理IP>
- 等待“六项就绪”:
- Master `/readyz`=200、ES `/_cluster/health`=200、Prometheus TCP 可达、Grafana `/api/health`=200、Alertmanager `/api/v2/status`=200、Kibana `/api/status` level=available;
- 将各服务 overlay IP 写入 `private/argus/etc/<域名>`,Reload Bind9 与 Nginx;
- 写出 `cluster-info.env`(含 `BINDIP/FTPIP/SWARM_JOIN_TOKEN_{WORKER,MANAGER}/SWARM_MANAGER_ADDR`);
- 写出 `cluster-info.env`(含 `SWARM_JOIN_TOKEN_{WORKER,MANAGER}/SWARM_MANAGER_ADDR`;compose 架构下不再依赖 BINDIP/FTPIP);
- 生成 `安装报告_YYYYMMDD-HHMMSS.md`(端口、健康检查摘要与提示)。
看到什么才算成功:
@@ -79,7 +79,7 @@ export SWARM_MANAGER_ADDR=<本机管理IP>
- `安装报告_…md` 中各项 HTTP 检查为 200/available;
- `cluster-info.env` 包含五个关键键:
- `SWARM_MANAGER_ADDR=...`
- `BINDIP=10.x.x.x` `FTPIP=10.x.x.x`
- `SWARM_MANAGER_ADDR=...` `SWARM_JOIN_TOKEN_*=...`
- `SWARM_JOIN_TOKEN_WORKER=SWMTKN-...`
- `SWARM_JOIN_TOKEN_MANAGER=SWMTKN-...`

View File

@@ -88,23 +88,15 @@ for i in $(seq 1 "$RETRIES"); do
done
[[ $ok -ge 6 ]] || err "部分服务未就绪(可稍后重试 selfcheck)"
# Resolve overlay IPs
bind_c=argus-bind-sys; ftp_c=argus-ftp
BINDIP=$(docker inspect -f '{{ (index .NetworkSettings.Networks "'$NET_NAME'").IPAddress }}' "$bind_c" 2>/dev/null || true)
FTPIP=$(docker inspect -f '{{ (index .NetworkSettings.Networks "'$NET_NAME'").IPAddress }}' "$ftp_c" 2>/dev/null || true)
info "解析 overlay IP: BINDIP=${BINDIP:-<empty>} FTPIP=${FTPIP:-<empty>}"
# Swarm join tokens
TOKEN_WORKER=$(docker swarm join-token -q worker 2>/dev/null || echo "")
TOKEN_MANAGER=$(docker swarm join-token -q manager 2>/dev/null || echo "")
# cluster-info.env
# cluster-info.env(compose 场景下不再依赖 BINDIP/FTPIP)
CI="$PKG_ROOT/cluster-info.env"
info "写入 cluster-info.env (manager/token/IP)"
info "写入 cluster-info.env (manager/token)"
{
echo "SWARM_MANAGER_ADDR=${SWARM_MANAGER_ADDR:-}"
echo "BINDIP=${BINDIP:-}"
echo "FTPIP=${FTPIP:-}"
echo "SWARM_JOIN_TOKEN_WORKER=${TOKEN_WORKER:-}"
echo "SWARM_JOIN_TOKEN_MANAGER=${TOKEN_MANAGER:-}"
} > "$CI"
@@ -131,10 +123,6 @@ RPT="$PKG_ROOT/安装报告_${ts}.md"
echo "- JOIN_TOKEN_WORKER=${TOKEN_WORKER:-}"
echo "- JOIN_TOKEN_MANAGER=${TOKEN_MANAGER:-}"
echo
echo "## Overlay IPs"
echo "- BINDIP=${BINDIP:-}"
echo "- FTPIP=${FTPIP:-}"
echo
echo "## 健康检查(简要)"
echo "- master/readyz=$(code http://127.0.0.1:${MASTER_PORT:-32300}/readyz)"
echo "- es/_cluster/health=$(code http://127.0.0.1:${ES_HTTP_PORT:-9200}/_cluster/health)"
@@ -146,30 +134,4 @@ RPT="$PKG_ROOT/安装报告_${ts}.md"
info "已生成报告: $RPT"
info "安装完成。可将 cluster-info.env 分发给 Client-GPU 安装方。"
# 写入域名→overlay IP 并热更新 Bind/Nginx
ETC_DIR="$PKG_ROOT/private/argus/etc"; mkdir -p "$ETC_DIR"
declare -A MAP
MAP[web-frontend]=web.argus.com
MAP[argus-grafana]=grafana.metric.argus.com
MAP[argus-prometheus]=prom.metric.argus.com
MAP[argus-kibana-sys]=kibana.log.argus.com
MAP[argus-alertmanager]=alertmanager.alert.argus.com
MAP[argus-master-sys]=master.argus.com
changed=0
for cname in "${!MAP[@]}"; do
domain="${MAP[$cname]}"; fpath="$ETC_DIR/$domain"
ip=$(docker inspect -f '{{ (index .NetworkSettings.Networks "'$NET_NAME'").IPAddress }}' "$cname" 2>/dev/null || true)
[[ -z "$ip" ]] && { echo "[DNS-FIX][WARN] $domain: container $cname no overlay IP yet"; continue; }
cur=$(cat "$fpath" 2>/dev/null || echo "")
if [[ "$cur" != "$ip" ]]; then
echo "$ip" > "$fpath"; echo "[DNS-FIX][SET] $domain = $ip (was: ${cur:-<empty>})"; changed=1
else
echo "[DNS-FIX][OK] $domain already $ip"
fi
done
if [[ $changed -eq 1 ]]; then
docker exec argus-bind-sys /usr/local/bin/reload-bind9.sh >/dev/null 2>&1 || docker exec argus-bind-sys rndc reload >/dev/null 2>&1 || true
sleep 1
fi
docker exec argus-web-proxy nginx -t >/dev/null 2>&1 && docker exec argus-web-proxy nginx -s reload >/dev/null 2>&1 || true