[#37] swarm 部署优化

2025-11-04 09:34:42 +08:00
parent 4ed5c64804
commit c8279997a4
7 changed files with 234 additions and 29 deletions
--- a/deployment/build/templates/scripts/server-diagnose.sh
+++ b/deployment/build/templates/scripts/server-diagnose.sh
@@ -6,10 +6,16 @@ ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"

 ENV_FILE="$ROOT/compose/.env"; [[ -f "$ENV_FILE" ]] && set -a && source "$ENV_FILE" && set +a

-mkdir -p "$ROOT/logs"
 ts="$(date -u +%Y%m%d-%H%M%SZ)"
-DETAILS="$ROOT/logs/diagnose_details_${ts}.log"
-ERRORS="$ROOT/logs/diagnose_error_${ts}.log"
+LOG_DIR="$ROOT/logs"
+mkdir -p "$LOG_DIR" || true
+# Fallback to /tmp when logs dir is not writable
+if ! ( : > "$LOG_DIR/.w" 2>/dev/null ); then
+  LOG_DIR="/tmp/argus-logs"
+  mkdir -p "$LOG_DIR" || true
+fi
+DETAILS="$LOG_DIR/diagnose_details_${ts}.log"
+ERRORS="$LOG_DIR/diagnose_error_${ts}.log"
 : > "$DETAILS"; : > "$ERRORS"

 logd() { echo "$(date '+%F %T') $*" >> "$DETAILS"; }
@@ -83,6 +89,25 @@ logd "Web-Proxy 8083: $(http_code "http://localhost:${WEB_PROXY_PORT_8083:-8083}
 logd "Web-Proxy 8084 CORS: ${cors8084}"
 logd "Web-Proxy 8085 CORS: ${cors8085}"

+# Overlay network diagnostics
+section OVERLAY-NET
+if docker network inspect "${OVERLAY_NET_NAME:-argus-sys-net}" >/dev/null 2>&1; then
+  logd "overlay present: ${OVERLAY_NET_NAME:-argus-sys-net}"
+  docker network inspect "${OVERLAY_NET_NAME:-argus-sys-net}" | sed -n '1,60p' >> "$DETAILS" 2>/dev/null || true
+else
+  append_err "[overlay][network] missing ${OVERLAY_NET_NAME:-argus-sys-net}"
+fi
+
+# Domain resolution & reachability from inside web-proxy (bind-backed)
+section DOMAIN
+for d in master.argus.com ftp.argus.com kibana.argus.com es.log.argus.com prom.argus.com grafana.argus.com; do
+  logd "getent $d (web-proxy):"
+  docker exec argus-web-proxy sh -lc "getent hosts $d || true" >> "$DETAILS" 2>&1 || true
+done
+logd "HTTP (web-proxy): master.readyz=$(docker exec argus-web-proxy sh -lc \"curl -s -o /dev/null -w '%{http_code}' http://master.argus.com:3000/readyz\" 2>/dev/null || echo 000)"
+logd "HTTP (web-proxy): es.health=$(docker exec argus-web-proxy sh -lc \"curl -s -o /dev/null -w '%{http_code}' http://es.log.argus.com:9200/_cluster/health\" 2>/dev/null || echo 000)"
+logd "HTTP (web-proxy): kibana.status=$(docker exec argus-web-proxy sh -lc \"curl -s -o /dev/null -w '%{http_code}' http://kibana.argus.com:5601/api/status\" 2>/dev/null || echo 000)"
+
 # FTP share writability (container perspective)
 section FTP-SHARE
 docker exec argus-ftp sh -lc 'ls -ld /private/argus/ftp /private/argus/ftp/share; test -w /private/argus/ftp/share && echo "write:OK" || echo "write:FAIL"' >> "$DETAILS" 2>&1 || true
@@ -110,8 +135,13 @@ sort -u -o "$ERRORS" "$ERRORS"
 echo "Diagnostic details -> $DETAILS"
 echo "Detected errors   -> $ERRORS"

-# maintain latest symlinks for convenience
-ln -sfn "$(basename "$DETAILS")" "$ROOT/logs/diagnose_details.log" 2>/dev/null || cp "$DETAILS" "$ROOT/logs/diagnose_details.log" 2>/dev/null || true
-ln -sfn "$(basename "$ERRORS")"  "$ROOT/logs/diagnose_error.log"   2>/dev/null || cp "$ERRORS"  "$ROOT/logs/diagnose_error.log"   2>/dev/null || true
+if [[ "$LOG_DIR" == "$ROOT/logs" ]]; then
+  # maintain latest symlinks when writing under package logs
+  ln -sfn "$(basename "$DETAILS")" "$ROOT/logs/diagnose_details.log" 2>/dev/null || cp "$DETAILS" "$ROOT/logs/diagnose_details.log" 2>/dev/null || true
+  ln -sfn "$(basename "$ERRORS")"  "$ROOT/logs/diagnose_error.log"   2>/dev/null || cp "$ERRORS"  "$ROOT/logs/diagnose_error.log"   2>/dev/null || true
+else
+  echo "Diagnostic details -> $DETAILS"
+  echo "Detected errors   -> $ERRORS"
+fi

 exit 0
--- a/deployment/build/templates/scripts/server-install.sh
+++ b/deployment/build/templates/scripts/server-install.sh
@@ -30,17 +30,7 @@ prepare_env() {
  if [[ -f "$ENV_FILE" ]]; then log ".env exists, keep as-is"; return; fi
  [[ -f "$ENV_TEMPLATE" ]] || { err "missing $ENV_TEMPLATE"; exit 1; }
  cp "$ENV_TEMPLATE" "$ENV_FILE"
-  # auto-assign ports if busy
-  for key in MASTER_PORT ES_HTTP_PORT KIBANA_PORT NODE_A_PORT NODE_B_PORT PROMETHEUS_PORT GRAFANA_PORT ALERTMANAGER_PORT \
-             WEB_PROXY_PORT_8080 WEB_PROXY_PORT_8081 WEB_PROXY_PORT_8082 WEB_PROXY_PORT_8083 WEB_PROXY_PORT_8084 WEB_PROXY_PORT_8085 \
-             FTP_PORT FTP_DATA_PORT; do
-    val=$(grep -E "^${key}=" "$ENV_FILE" | tail -1 | cut -d= -f2)
-    new=$(find_free_port "$val") || true
-    if [[ -n "${new:-}" && "$new" != "$val" ]]; then
-      sed -i "s/^${key}=.*/${key}=${new}/" "$ENV_FILE"
-      log "port ${key} busy -> ${new}"
-    fi
-  done
+  # overlay 模式下，避免为不同服务分配到同一新端口；保持模板端口不做自动改写
 }

 prepare_data_dirs() {
@@ -63,6 +53,43 @@ prepare_data_dirs() {
      "$PKG_ROOT/private/argus/metric/grafana/data/dashboards" \
      "$PKG_ROOT/private/argus/alert/alertmanager" \
      "$PKG_ROOT/private/argus/metric/ftp/share"
+    # non-root: relax permissions to avoid container UID mismatch blocking writes
+    chmod -R a+rwx "$PKG_ROOT/private/argus" 2>/dev/null || true
+  fi
+}
+
+ensure_swarm_and_overlay() {
+  local net_name="${OVERLAY_NET_NAME:-argus-sys-net}"
+  # Require swarm active
+  local state
+  state=$(docker info --format '{{.Swarm.LocalNodeState}}' 2>/dev/null || echo "")
+  if [[ "$state" != "active" ]]; then
+    err "Docker Swarm is not active. On this host run:"
+    err "  docker swarm init --advertise-addr <this_host_ip>"
+    exit 1
+  fi
+  # Create attachable overlay if missing
+  if ! docker network inspect "$net_name" >/dev/null 2>&1; then
+    log "creating attachable overlay network: $net_name"
+    docker network create --driver overlay --attachable "$net_name" >/dev/null
+  fi
+}
+
+bootstrap_dns_conf() {
+  local etc_dir="$PKG_ROOT/private/argus/etc"
+  mkdir -p "$etc_dir"
+  local dns_file="$etc_dir/dns.conf"
+  if [[ ! -s "$dns_file" ]]; then
+    # detect host primary IP
+    local host_ip
+    host_ip=$(ip route get 1.1.1.1 2>/dev/null | awk '/src/ {print $7; exit}')
+    [[ -z "$host_ip" ]] && host_ip=$(hostname -I 2>/dev/null | awk '{print $1}')
+    if [[ -n "$host_ip" ]]; then
+      echo "$host_ip" > "$dns_file"
+      log "wrote initial dns.conf with host IP: $host_ip"
+    else
+      err "failed to determine host IP for dns.conf; please edit $dns_file manually"
+    fi
  fi
 }

@@ -75,6 +102,8 @@ load_images() {

 bring_up() {
  log "starting services via compose"
+  ensure_swarm_and_overlay
+  bootstrap_dns_conf
  local ov="$PKG_ROOT/compose/docker-compose.os-compat.override.yml"
  if [[ ! -f "$ov" ]]; then
    cat > "$ov" <<'YAML'
@@ -182,6 +211,7 @@ selfcheck() {
 }

 main() {
+  mkdir -p "$PKG_ROOT/logs"
  prepare_env
  prepare_data_dirs
  load_images
--- a/deployment/build/templates/scripts/server-selfcheck.sh
+++ b/deployment/build/templates/scripts/server-selfcheck.sh
@@ -13,23 +13,31 @@ wait_http() { local url="$1"; local attempts=${2:-120}; local i=1; while ((i<=at
 code_for() { curl -s -o /dev/null -w "%{http_code}" "$1" || echo 000; }
 header_val() { curl -s -D - -o /dev/null "$@" | awk -F': ' 'BEGIN{IGNORECASE=1}$1=="Access-Control-Allow-Origin"{gsub("\r","",$2);print $2}'; }

-mkdir -p "$ROOT/logs"
-OUT_JSON="$ROOT/logs/selfcheck.json"
+LOG_DIR="$ROOT/logs"
+mkdir -p "$LOG_DIR" || true
+OUT_JSON="$LOG_DIR/selfcheck.json"
 tmp=$(mktemp)

 ok=1

-log "checking Elasticsearch"
-if curl -fsS "http://localhost:${ES_HTTP_PORT:-9200}/_cluster/health?wait_for_status=yellow&timeout=1s" >/dev/null 2>&1; then es_ok=true; else es_ok=false; ok=0; fi
+log "checking overlay network"
+net_ok=false
+if docker network inspect "${OVERLAY_NET_NAME:-argus-sys-net}" >/dev/null 2>&1; then
+  if docker network inspect "${OVERLAY_NET_NAME:-argus-sys-net}" | grep -q '"Driver": "overlay"'; then net_ok=true; fi
+fi
+[[ "$net_ok" == true ]] || ok=0

-log "checking Kibana"
-kb_code=$(curl -s -o /dev/null -w "%{http_code}" "http://localhost:${KIBANA_PORT:-5601}/api/status" || echo 000)
+log "checking Elasticsearch (via domain inside web-proxy)"
+if docker exec argus-web-proxy sh -lc "curl -fsS http://es.log.argus.com:9200/_cluster/health?wait_for_status=yellow&timeout=1s" >/dev/null 2>&1; then es_ok=true; else es_ok=false; ok=0; fi
+
+log "checking Kibana (via domain inside web-proxy)"
+kb_code=$(docker exec argus-web-proxy sh -lc "curl -s -o /dev/null -w '%{http_code}' http://kibana.log.argus.com:5601/api/status" || echo 000)
 kb_ok=false
 if [[ "$kb_code" == "200" ]]; then body=$(curl -sS "http://localhost:${KIBANA_PORT:-5601}/api/status"); echo "$body" | grep -q '"level":"available"' && kb_ok=true; fi
 [[ "$kb_ok" == true ]] || ok=0

-log "checking Master"
-wait_http "http://localhost:${MASTER_PORT:-32300}/readyz" 60 || ok=0
+log "checking Master (via domain inside web-proxy)"
+if docker exec argus-web-proxy sh -lc "curl -fsS http://master.argus.com:3000/readyz" >/dev/null 2>&1; then true; else ok=0; fi

 log "checking FTP"
 if docker ps --format '{{.Names}}' | grep -q '^argus-ftp$'; then
@@ -71,11 +79,17 @@ cat > "$tmp" <<JSON
  "grafana": $gf_ok,
  "alertmanager": true,
  "web_proxy": $wp_ok,
+  "overlay_net": $net_ok,
  "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)"
 }
 JSON

-mv "$tmp" "$OUT_JSON"
+if ! mv "$tmp" "$OUT_JSON" 2>/dev/null; then
+  # fallback when logs dir not writable (no sudo allowed)
+  OUT_JSON="/tmp/selfcheck_$(date -u +%Y%m%d-%H%M%SZ).json"
+  cp "$tmp" "$OUT_JSON"
+  log "selfcheck.json written to $OUT_JSON (logs dir not writable)"
+fi
 if [[ "$ok" == 1 ]]; then
  log "selfcheck OK"
  exit 0