dev_1.0.0_yuyr_2：重新提交 PR，增加 master/agent 以及系统集成测试 (#17)

Reviewed-on: #17 Reviewed-by: sundapeng <sundp@mail.zgclab.edu.cn> Reviewed-by: xuxt <xuxt@zgclab.edu.cn>
2025-10-11 15:04:46 +08:00
parent 8a38d3d0b2
commit 1e5e91b193
105 changed files with 6096 additions and 107 deletions
--- a/src/log/elasticsearch/build/Dockerfile
+++ b/src/log/elasticsearch/build/Dockerfile
@@ -3,10 +3,29 @@ FROM docker.elastic.co/elasticsearch/elasticsearch:8.13.4
 # 切换到 root 用户进行系统级安装
 USER root

-# 修改elasticsearch用户的UID和GID
-RUN usermod -u 2133 elasticsearch && \
-    groupmod -g 2015 elasticsearch && \
-    chown -R elasticsearch:elasticsearch /usr/share/elasticsearch
+ARG ARGUS_BUILD_UID=2133
+ARG ARGUS_BUILD_GID=2015
+
+ENV ARGUS_BUILD_UID=${ARGUS_BUILD_UID} \
+    ARGUS_BUILD_GID=${ARGUS_BUILD_GID}
+
+# 调整 elasticsearch 用户与用户组 ID 以匹配宿主机配置
+RUN set -eux; \
+    current_gid="$(getent group elasticsearch | awk -F: '{print $3}')"; \
+    if [ -z "$current_gid" ]; then \
+        groupadd -g "${ARGUS_BUILD_GID}" elasticsearch; \
+    elif [ "$current_gid" != "${ARGUS_BUILD_GID}" ]; then \
+        groupmod -g "${ARGUS_BUILD_GID}" elasticsearch; \
+    fi; \
+    if id elasticsearch >/dev/null 2>&1; then \
+        current_uid="$(id -u elasticsearch)"; \
+        if [ "$current_uid" != "${ARGUS_BUILD_UID}" ]; then \
+            usermod -u "${ARGUS_BUILD_UID}" elasticsearch; \
+        fi; \
+    else \
+        useradd -m -u "${ARGUS_BUILD_UID}" -g "${ARGUS_BUILD_GID}" elasticsearch; \
+    fi; \
+    chown -R "${ARGUS_BUILD_UID}:${ARGUS_BUILD_GID}" /usr/share/elasticsearch

 # 设置构建参数
 ARG USE_INTRANET=false
--- a/src/log/fluent-bit/build/etc/parsers.conf
+++ b/src/log/fluent-bit/build/etc/parsers.conf
@@ -25,3 +25,5 @@
    Regex  ^(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+(?<level>\w+)\s+(?<message>.*)$
    Time_Key    timestamp
    Time_Format %Y-%m-%d %H:%M:%S
+    Time_Offset +0800
+    Time_Keep On
--- a/src/log/kibana/build/Dockerfile
+++ b/src/log/kibana/build/Dockerfile
@@ -3,10 +3,29 @@ FROM docker.elastic.co/kibana/kibana:8.13.4
 # 切换到 root 用户进行系统级安装
 USER root

-# 修改kibana用户的UID和GID
-RUN usermod -u 2133 kibana && \
-    groupmod -g 2015 kibana && \
-    chown -R kibana:kibana /usr/share/kibana
+ARG ARGUS_BUILD_UID=2133
+ARG ARGUS_BUILD_GID=2015
+
+ENV ARGUS_BUILD_UID=${ARGUS_BUILD_UID} \
+    ARGUS_BUILD_GID=${ARGUS_BUILD_GID}
+
+# 调整 kibana 用户与用户组 ID 以匹配宿主机配置
+RUN set -eux; \
+    current_gid="$(getent group kibana | awk -F: '{print $3}')"; \
+    if [ -z "$current_gid" ]; then \
+        groupadd -g "${ARGUS_BUILD_GID}" kibana; \
+    elif [ "$current_gid" != "${ARGUS_BUILD_GID}" ]; then \
+        groupmod -g "${ARGUS_BUILD_GID}" kibana; \
+    fi; \
+    if id kibana >/dev/null 2>&1; then \
+        current_uid="$(id -u kibana)"; \
+        if [ "$current_uid" != "${ARGUS_BUILD_UID}" ]; then \
+            usermod -u "${ARGUS_BUILD_UID}" kibana; \
+        fi; \
+    else \
+        useradd -m -u "${ARGUS_BUILD_UID}" -g "${ARGUS_BUILD_GID}" kibana; \
+    fi; \
+    chown -R "${ARGUS_BUILD_UID}:${ARGUS_BUILD_GID}" /usr/share/kibana

 # 设置构建参数
 ARG USE_INTRANET=false
--- a/src/log/tests/docker-compose.yml
+++ b/src/log/tests/docker-compose.yml
@@ -17,6 +17,7 @@ services:
      interval: 10s
      timeout: 5s
      retries: 30
+    restart: always

  kibana:
    build:
@@ -73,13 +74,11 @@ services:
      interval: 15s
      timeout: 10s
      retries: 30
+    restart: always

  bind9:
      image: argus-bind9:latest
-      ports:
-        - "53:53/tcp"
-        - "53:53/udp"
      volumes:
        - ./private/argus:/private/argus/
-      restart: unless-stopped
+      restart: always

--- a/src/log/tests/scripts/01_bootstrap.sh
+++ b/src/log/tests/scripts/01_bootstrap.sh
@@ -1,6 +1,10 @@
 #!/usr/bin/env bash
 set -euo pipefail
 root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd)"
+project_root="$(cd "$root/../../.." && pwd)"
+
+source "$project_root/scripts/common/build_user.sh"
+load_build_user

 # 创建新的private目录结构 (基于argus目录结构)
 echo "[INFO] Creating private directory structure for supervisor-based containers..."
@@ -11,9 +15,9 @@ mkdir -p "$root/private/argus/etc/"

 # 设置数据目录权限（ES 和 Kibana 容器都使用 UID 1000）
 echo "[INFO] Setting permissions for data directories..."
-sudo chown -R 2133:2015 "$root/private/argus/log/elasticsearch" 2>/dev/null || true
-sudo chown -R 2133:2015 "$root/private/argus/log/kibana" 2>/dev/null || true
-sudo chown -R 2133:2015 "$root/private/argus/etc" 2>/dev/null || true
+chown -R "${ARGUS_BUILD_UID}:${ARGUS_BUILD_GID}" "$root/private/argus/log/elasticsearch" 2>/dev/null || true
+chown -R "${ARGUS_BUILD_UID}:${ARGUS_BUILD_GID}" "$root/private/argus/log/kibana" 2>/dev/null || true
+chown -R "${ARGUS_BUILD_UID}:${ARGUS_BUILD_GID}" "$root/private/argus/etc" 2>/dev/null || true

 echo "[INFO] Supervisor-based containers will manage their own scripts and configurations"

--- a/src/log/tests/scripts/03_send_test_host01.sh
+++ b/src/log/tests/scripts/03_send_test_host01.sh
@@ -4,8 +4,22 @@ set -euo pipefail
 # 获取fluent-bit-host01容器名称
 container_name="logging-mvp-fluent-bit-host01-1"

-# 检查容器是否存在并运行
-if ! docker ps | grep -q "$container_name"; then
+wait_for_container() {
+    local name="$1"
+    local attempts=30
+    local delay=5
+    local i
+    for ((i = 1; i <= attempts; i++)); do
+        if docker ps --format '{{.Names}}' | grep -Fx "$name" >/dev/null; then
+            return 0
+        fi
+        echo "[INFO] 等待容器 $name 启动中... ($i/$attempts)"
+        sleep "$delay"
+    done
+    return 1
+}
+
+if ! wait_for_container "$container_name"; then
    echo "[ERROR] Fluent Bit容器 $container_name 未运行"
    exit 1
 fi
@@ -28,4 +42,4 @@ STACK"

 echo "[OK] 已通过docker exec写入测试日志到 host01 容器内："
 echo " - /logs/train/train-demo.log"
-echo " - /logs/infer/infer-demo.log"
+echo " - /logs/infer/infer-demo.log"
--- a/src/log/tests/scripts/03_send_test_host02.sh
+++ b/src/log/tests/scripts/03_send_test_host02.sh
@@ -4,8 +4,22 @@ set -euo pipefail
 # 获取fluent-bit-host02容器名称
 container_name="logging-mvp-fluent-bit-host02-1"

-# 检查容器是否存在并运行
-if ! docker ps | grep -q "$container_name"; then
+wait_for_container() {
+    local name="$1"
+    local attempts=30
+    local delay=5
+    local i
+    for ((i = 1; i <= attempts; i++)); do
+        if docker ps --format '{{.Names}}' | grep -Fx "$name" >/dev/null; then
+            return 0
+        fi
+        echo "[INFO] 等待容器 $name 启动中... ($i/$attempts)"
+        sleep "$delay"
+    done
+    return 1
+}
+
+if ! wait_for_container "$container_name"; then
    echo "[ERROR] Fluent Bit容器 $container_name 未运行"
    exit 1
 fi
@@ -24,4 +38,4 @@ docker exec "$container_name" sh -c "printf '%s INFO [host02] inference complete

 echo "[OK] 已通过docker exec写入测试日志到 host02 容器内："
 echo " - /logs/train/train-demo.log"
-echo " - /logs/infer/infer-demo.log"
+echo " - /logs/infer/infer-demo.log"
--- a/src/log/tests/scripts/04_query_es.sh
+++ b/src/log/tests/scripts/04_query_es.sh
@@ -1,7 +1,42 @@
 #!/usr/bin/env bash
 set -euo pipefail
+
+# ES endpoint and wait strategy
 ES="${ES:-http://localhost:9200}"
+es_wait_attempts="${ES_WAIT_ATTEMPTS:-60}"      # total attempts to wait for ES
+es_wait_interval="${ES_WAIT_INTERVAL:-2}"        # seconds between attempts
+
 echo "[i] 查询 ES 端点：$ES"
+
+wait_for_es() {
+  local attempt=1
+  while (( attempt <= es_wait_attempts )); do
+    # 等待集群达到至少 yellow 状态；请求失败则重试
+    if curl -fsS "$ES/_cluster/health?wait_for_status=yellow&timeout=1s" >/dev/null 2>&1; then
+      echo "[ok] Elasticsearch 已就绪 (attempt=${attempt}/${es_wait_attempts})"
+      return 0
+    fi
+    echo "[..] 等待 Elasticsearch 可用中 (${attempt}/${es_wait_attempts})"
+    sleep "${es_wait_interval}"
+    (( attempt++ ))
+  done
+  echo "[err] Elasticsearch 在 ${es_wait_attempts} 次尝试后仍不可用"
+  return 1
+}
+
+safe_count() {
+  # 对缺失索引返回 0，避免 404 触发失败
+  local pattern="$1"
+  local json
+  json=$(curl -fsS "$ES/${pattern}/_count?ignore_unavailable=true&allow_no_indices=true" 2>/dev/null || echo '{}')
+  echo "$json" | sed -E 's/.*"count":([0-9]+).*/\1/' | awk 'NF{print $0;exit} END{if(NR==0)print 0}'
+}
+
+wait_for_es
+
+# 列出相关索引（可能为空，允许）
 curl -fsS "$ES/_cat/indices?v" | egrep 'train-|infer-|logstash' || true
-printf "train-* 计数："; curl -fsS "$ES/train-*/_count" | sed -E 's/.*"count":([0-9]+).*/\1/'; echo
-printf "infer-* 计数："; curl -fsS "$ES/infer-*/_count" | sed -E 's/.*"count":([0-9]+).*/\1/'; echo
+
+# 打印计数，缺失索引按 0 处理
+printf "train-* 计数："; safe_count "train-*"; echo
+printf "infer-* 计数："; safe_count "infer-*"; echo
--- a/src/log/tests/scripts/e2e_test.sh
+++ b/src/log/tests/scripts/e2e_test.sh
@@ -19,7 +19,7 @@ get_log_count() {
 # 函数：等待服务就绪
 wait_for_services() {
    echo "[INFO] Waiting for all services to be ready..."
-    local max_attempts=60
+    local max_attempts=${SERVICE_WAIT_ATTEMPTS:-120}
    local attempt=1

    while [ $attempt -le $max_attempts ]; do