dev_1.0.0_yuyr 完成 log和bind模块开发部署测试 (#8)

- [x] 完成log模块镜像构建、本地端到端写日志——收集——查询流程； - [x] 完成bind模块构建； - [x] 内置域名IP自动更新脚本，使用 /private/argus/etc目录下文件进行同步，容器启动时自动写IP，定时任务刷新更新DNS服务器IP和DNS规则； Co-authored-by: root <root@curious.host.com> Reviewed-on: #8 Reviewed-by: sundapeng <sundp@mail.zgclab.edu.cn>
2025-09-22 16:39:38 +08:00
parent 26e1c964ed
commit 8a38d3d0b2
48 changed files with 2744 additions and 0 deletions
--- a/src/log/kibana/build/Dockerfile
+++ b/src/log/kibana/build/Dockerfile
@@ -0,0 +1,60 @@
+FROM docker.elastic.co/kibana/kibana:8.13.4
+
+# 切换到 root 用户进行系统级安装
+USER root
+
+# 修改kibana用户的UID和GID
+RUN usermod -u 2133 kibana && \
+    groupmod -g 2015 kibana && \
+    chown -R kibana:kibana /usr/share/kibana
+
+# 设置构建参数
+ARG USE_INTRANET=false
+
+# 配置内网 apt 源 (如果指定了内网选项)
+RUN if [ "$USE_INTRANET" = "true" ]; then \
+        echo "Configuring intranet apt sources..." && \
+        cp /etc/apt/sources.list /etc/apt/sources.list.bak && \
+        echo "deb [trusted=yes]  http://10.68.64.1/ubuntu2204/ jammy main" > /etc/apt/sources.list && \
+        echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/99disable-ssl-check && \
+        echo 'Acquire::https::Verify-Host "false";' >> /etc/apt/apt.conf.d/99disable-ssl-check; \
+    fi
+
+# 安装 supervisor, net-tools, vim
+RUN apt-get update && \
+    apt-get install -y supervisor net-tools inetutils-ping vim && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/*
+
+# 配置部署时使用的apt源
+RUN if [ "$USE_INTRANET" = "true" ]; then \
+        echo "deb [trusted=yes] https://10.92.132.52/mirrors/ubuntu2204/ jammy main" > /etc/apt/sources.list; \
+    fi
+
+# 创建 supervisor 日志目录
+RUN mkdir -p /var/log/supervisor
+
+
+# 复制 supervisor 配置文件
+COPY src/log/kibana/build/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
+
+# 复制启动脚本
+COPY src/log/kibana/build/start-kibana-supervised.sh /usr/local/bin/start-kibana-supervised.sh
+COPY src/log/kibana/build/kibana-post-start.sh /usr/local/bin/kibana-post-start.sh
+RUN chmod +x /usr/local/bin/start-kibana-supervised.sh /usr/local/bin/kibana-post-start.sh
+
+# 复制DNS监控脚本
+COPY src/log/kibana/build/dns-monitor.sh /usr/local/bin/dns-monitor.sh
+RUN chmod +x /usr/local/bin/dns-monitor.sh
+
+# kibana需要用到 /root/.config/puppeteer 路径
+RUN chmod 777 /root
+
+# 保持 root 用户，由 supervisor 管理用户切换
+USER root
+
+# 暴露端口
+EXPOSE 5601
+
+# 使用 supervisor 作为入口点
+CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
--- a/src/log/kibana/build/dns-monitor.sh
+++ b/src/log/kibana/build/dns-monitor.sh
@@ -0,0 +1 @@
+../../../bind/build/dns-monitor.sh
--- a/src/log/kibana/build/kibana-post-start.sh
+++ b/src/log/kibana/build/kibana-post-start.sh
@@ -0,0 +1,146 @@
+#!/bin/bash
+set -euo pipefail
+
+ES_HOST="${ELASTICSEARCH_HOSTS:-http://es:9200}"
+KB_HOST="http://localhost:5601"
+
+echo "[INFO] Starting Kibana post-start configuration..."
+
+# 等待 Elasticsearch 可用
+wait_for_elasticsearch() {
+    echo "[INFO] Waiting for Elasticsearch..."
+    local max_attempts=60
+    local attempt=1
+
+    while [ $attempt -le $max_attempts ]; do
+        if curl -fs "$ES_HOST/_cluster/health" >/dev/null 2>&1; then
+            echo "[OK] Elasticsearch is available"
+            return 0
+        fi
+        echo "    Waiting for ES... ($attempt/$max_attempts)"
+        sleep 5
+        ((attempt++))
+    done
+
+    echo "[ERROR] Elasticsearch timeout"
+    return 1
+}
+
+# 等待 Kibana 可用
+wait_for_kibana() {
+    echo "[INFO] Waiting for Kibana..."
+    local max_attempts=120
+    local attempt=1
+
+    while [ $attempt -le $max_attempts ]; do
+        if curl -fs "$KB_HOST/api/status" >/dev/null 2>&1; then
+            local status=$(curl -s "$KB_HOST/api/status" | grep -o '"level":"available"' || echo "")
+            if [ -n "$status" ]; then
+                echo "[OK] Kibana is available"
+                return 0
+            fi
+            echo "    Waiting for Kibana... ($attempt/$max_attempts, status: $status)"
+        else
+            echo "    Waiting for Kibana... ($attempt/$max_attempts, connection failed)"
+        fi
+        sleep 5
+        ((attempt++))
+    done
+
+    echo "[ERROR] Kibana timeout"
+    return 1
+}
+
+# 幂等设置索引副本数为0
+fix_replicas_idempotent() {
+    echo "[INFO] Checking and fixing index replicas..."
+
+    # 获取所有 train-* 和 infer-* 索引
+    local indices=$(curl -s "$ES_HOST/_cat/indices/train-*,infer-*?h=index" 2>/dev/null || echo "")
+
+    if [ -z "$indices" ]; then
+        echo "[INFO] No train-*/infer-* indices found, skipping replica adjustment"
+        return 0
+    fi
+
+    for idx in $indices; do
+        # 检查当前副本数
+        local current_replicas=$(curl -s "$ES_HOST/$idx/_settings" | grep -o '"number_of_replicas":"[^"]*"' | cut -d'"' -f4 || echo "")
+
+        if [ "$current_replicas" != "0" ]; then
+            echo "[INFO] Setting replicas to 0 for index: $idx (current: $current_replicas)"
+            curl -fsS -X PUT "$ES_HOST/$idx/_settings" \
+                -H 'Content-Type: application/json' \
+                -d '{"index":{"number_of_replicas":0}}' >/dev/null || {
+                echo "[WARN] Failed to set replicas for $idx"
+                continue
+            }
+            echo "[OK] Updated replicas for $idx"
+        else
+            echo "[INFO] Index $idx already has 0 replicas, skipping"
+        fi
+    done
+}
+
+# 幂等创建数据视图
+create_data_views_idempotent() {
+    echo "[INFO] Checking and creating data views..."
+
+    # 检查是否存在匹配的索引
+    local train_indices=$(curl -s "$ES_HOST/_cat/indices/train-*?h=index" 2>/dev/null | wc -l || echo "0")
+    local infer_indices=$(curl -s "$ES_HOST/_cat/indices/infer-*?h=index" 2>/dev/null | wc -l || echo "0")
+
+    # 创建 train 数据视图
+    if [ "$train_indices" -gt 0 ]; then
+        # 检查数据视图是否已存在
+        local train_exists=$(curl -s "$KB_HOST/api/data_views" -H 'kbn-xsrf: true' 2>/dev/null | grep '"title":"train-\*"' | wc -l )
+
+        if [ "$train_exists" -eq 0 ]; then
+            echo "[INFO] Creating data view for train-* indices"
+            curl -fsS -X POST "$KB_HOST/api/data_views/data_view" \
+                -H 'kbn-xsrf: true' \
+                -H 'Content-Type: application/json' \
+                -d '{"data_view":{"name":"train","title":"train-*","timeFieldName":"@timestamp"}}' \
+                >/dev/null && echo "[OK] Created train data view" || echo "[WARN] Failed to create train data view"
+        else
+            echo "[INFO] Train data view already exists, skipping"
+        fi
+    else
+        echo "[INFO] No train-* indices found, skipping train data view creation"
+    fi
+
+    # 创建 infer 数据视图
+    if [ "$infer_indices" -gt 0 ]; then
+        # 检查数据视图是否已存在
+        local infer_exists=$(curl -s "$KB_HOST/api/data_views" -H 'kbn-xsrf: true' 2>/dev/null | grep '"title":"infer-\*"' | wc -l )
+
+        if [ "$infer_exists" -eq 0 ]; then
+            echo "[INFO] Creating data view for infer-* indices"
+            curl -fsS -X POST "$KB_HOST/api/data_views/data_view" \
+                -H 'kbn-xsrf: true' \
+                -H 'Content-Type: application/json' \
+                -d '{"data_view":{"name":"infer","title":"infer-*","timeFieldName":"@timestamp"}}' \
+                >/dev/null && echo "[OK] Created infer data view" || echo "[WARN] Failed to create infer data view"
+        else
+            echo "[INFO] Infer data view already exists, skipping"
+        fi
+    else
+        echo "[INFO] No infer-* indices found, skipping infer data view creation"
+    fi
+}
+
+# 主逻辑
+main() {
+    # 等待服务可用
+    wait_for_elasticsearch || exit 1
+    wait_for_kibana || exit 1
+
+    # 执行幂等配置
+    fix_replicas_idempotent
+    create_data_views_idempotent
+
+    echo "[INFO] Kibana post-start configuration completed"
+}
+
+# 运行主逻辑
+main
--- a/src/log/kibana/build/start-kibana-supervised.sh
+++ b/src/log/kibana/build/start-kibana-supervised.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+set -euo pipefail
+
+echo "[INFO] Starting Kibana under supervisor..."
+
+mkdir -p /private/argus/log/kibana
+
+# 创建软链接到Kibana预期的数据目录
+if [ -L /usr/share/kibana/data ]; then
+    rm /usr/share/kibana/data
+elif [ -d /usr/share/kibana/data ]; then
+    rm -rf /usr/share/kibana/data
+fi
+
+ln -sf /private/argus/log/kibana /usr/share/kibana/data
+
+echo "[INFO] Data directory linked: /usr/share/kibana/data -> /private/argus/log/kibana"
+
+# 记录容器ip地址
+DOMAIN=kibana.log.argus.com
+IP=`ifconfig | grep -A 1 eth0 | grep inet | awk '{print $2}'`
+echo current IP: ${IP}
+echo ${IP} > /private/argus/etc/${DOMAIN}
+
+# 设置环境变量
+export ELASTICSEARCH_HOSTS="${ELASTICSEARCH_HOSTS:-"http://es:9200"}"
+
+echo "[INFO] Connecting to Elasticsearch at: $ELASTICSEARCH_HOSTS"
+
+# 启动后台配置任务
+echo "[INFO] Starting background post-start configuration..."
+/usr/local/bin/kibana-post-start.sh &
+
+echo "[INFO] Starting Kibana process..."
+
+# 启动原始的Kibana entrypoint
+exec /usr/local/bin/kibana-docker
--- a/src/log/kibana/build/supervisord.conf
+++ b/src/log/kibana/build/supervisord.conf
@@ -0,0 +1,39 @@
+[supervisord]
+nodaemon=true
+logfile=/var/log/supervisor/supervisord.log
+pidfile=/var/run/supervisord.pid
+user=root
+
+[program:kibana]
+command=/usr/local/bin/start-kibana-supervised.sh
+user=kibana
+stdout_logfile=/var/log/supervisor/kibana.log
+stderr_logfile=/var/log/supervisor/kibana_error.log
+autorestart=true
+startretries=3
+startsecs=30
+stopwaitsecs=30
+killasgroup=true
+stopasgroup=true
+
+[program:dns-monitor]
+command=/usr/local/bin/dns-monitor.sh
+user=root
+stdout_logfile=/var/log/supervisor/dns-monitor.log
+stderr_logfile=/var/log/supervisor/dns-monitor_error.log
+autorestart=true
+startretries=3
+startsecs=5
+stopwaitsecs=10
+killasgroup=true
+stopasgroup=true
+
+[unix_http_server]
+file=/var/run/supervisor.sock
+chmod=0700
+
+[supervisorctl]
+serverurl=unix:///var/run/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface