v1.0.0 发布合并回master (#52)
Co-authored-by: sundapeng <sundp@mail.zgclab.edu.cn> Co-authored-by: xuxt <xuxt@zgclab.edu.cn> Reviewed-on: #52
This commit was merged in pull request #52.
This commit is contained in:
5
src/log/.gitignore
vendored
Normal file
5
src/log/.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
|
||||
private/
|
||||
|
||||
|
||||
images/
|
||||
8
src/log/README.md
Normal file
8
src/log/README.md
Normal file
@@ -0,0 +1,8 @@
|
||||
|
||||
测试log模块开发
|
||||
|
||||
elasticsearch: 部署镜像构建及启动脚本（解决账号问题、挂载目录、使用supervisor守护）
|
||||
kibana: 镜像构建
|
||||
fluent-bit: 安装包，脚本准备， 交付给大鹏统一组织客户端侧安装流程
|
||||
init: EK初始化脚本：数据视图创建脚本等
|
||||
|
||||
75
src/log/elasticsearch/build/Dockerfile
Normal file
75
src/log/elasticsearch/build/Dockerfile
Normal file
@@ -0,0 +1,75 @@
|
||||
FROM docker.elastic.co/elasticsearch/elasticsearch:8.13.4
|
||||
|
||||
# 切换到 root 用户进行系统级安装
|
||||
USER root
|
||||
|
||||
ARG ARGUS_BUILD_UID=2133
|
||||
ARG ARGUS_BUILD_GID=2015
|
||||
|
||||
ENV ARGUS_BUILD_UID=${ARGUS_BUILD_UID} \
|
||||
ARGUS_BUILD_GID=${ARGUS_BUILD_GID}
|
||||
|
||||
# 调整 elasticsearch 用户与用户组 ID 以匹配宿主机配置
|
||||
RUN set -eux; \
|
||||
current_gid="$(getent group elasticsearch | awk -F: '{print $3}')"; \
|
||||
if [ -z "$current_gid" ]; then \
|
||||
groupadd -g "${ARGUS_BUILD_GID}" elasticsearch; \
|
||||
elif [ "$current_gid" != "${ARGUS_BUILD_GID}" ]; then \
|
||||
groupmod -g "${ARGUS_BUILD_GID}" elasticsearch; \
|
||||
fi; \
|
||||
if id elasticsearch >/dev/null 2>&1; then \
|
||||
current_uid="$(id -u elasticsearch)"; \
|
||||
if [ "$current_uid" != "${ARGUS_BUILD_UID}" ]; then \
|
||||
usermod -u "${ARGUS_BUILD_UID}" elasticsearch; \
|
||||
fi; \
|
||||
else \
|
||||
useradd -m -u "${ARGUS_BUILD_UID}" -g "${ARGUS_BUILD_GID}" elasticsearch; \
|
||||
fi; \
|
||||
chown -R "${ARGUS_BUILD_UID}:${ARGUS_BUILD_GID}" /usr/share/elasticsearch
|
||||
|
||||
# 设置构建参数
|
||||
ARG USE_INTRANET=false
|
||||
|
||||
# 配置内网 apt 源 (如果指定了内网选项)
|
||||
RUN if [ "$USE_INTRANET" = "true" ]; then \
|
||||
echo "Configuring intranet apt sources..." && \
|
||||
cp /etc/apt/sources.list /etc/apt/sources.list.bak && \
|
||||
echo "deb [trusted=yes] http://10.68.64.1/ubuntu2204/ jammy main" > /etc/apt/sources.list && \
|
||||
echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/99disable-ssl-check && \
|
||||
echo 'Acquire::https::Verify-Host "false";' >> /etc/apt/apt.conf.d/99disable-ssl-check; \
|
||||
fi
|
||||
|
||||
# 安装 supervisor, net-tools, vim
|
||||
RUN apt-get update && \
|
||||
apt-get install -y supervisor net-tools inetutils-ping vim && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 配置部署时使用的apt源
|
||||
RUN if [ "$USE_INTRANET" = "true" ]; then \
|
||||
echo "deb [trusted=yes] https://10.92.132.52/mirrors/ubuntu2204/ jammy main" > /etc/apt/sources.list; \
|
||||
fi
|
||||
|
||||
# 创建 supervisor 日志目录
|
||||
RUN mkdir -p /var/log/supervisor
|
||||
|
||||
|
||||
# 复制 supervisor 配置文件
|
||||
COPY src/log/elasticsearch/build/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
|
||||
|
||||
# 复制启动脚本
|
||||
COPY src/log/elasticsearch/build/start-es-supervised.sh /usr/local/bin/start-es-supervised.sh
|
||||
RUN chmod +x /usr/local/bin/start-es-supervised.sh
|
||||
|
||||
# 复制DNS监控脚本
|
||||
COPY src/log/elasticsearch/build/dns-monitor.sh /usr/local/bin/dns-monitor.sh
|
||||
RUN chmod +x /usr/local/bin/dns-monitor.sh
|
||||
|
||||
# 保持 root 用户，由 supervisor 管理用户切换
|
||||
USER root
|
||||
|
||||
# 暴露端口
|
||||
EXPOSE 9200 9300
|
||||
|
||||
# 使用 supervisor 作为入口点
|
||||
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
|
||||
1
src/log/elasticsearch/build/dns-monitor.sh
Symbolic link
1
src/log/elasticsearch/build/dns-monitor.sh
Symbolic link
@@ -0,0 +1 @@
|
||||
../../../bind/build/dns-monitor.sh
|
||||
32
src/log/elasticsearch/build/start-es-supervised.sh
Normal file
32
src/log/elasticsearch/build/start-es-supervised.sh
Normal file
@@ -0,0 +1,32 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "[INFO] Starting Elasticsearch under supervisor..."
|
||||
|
||||
# 创建数据目录并设置权限（如果不存在）
|
||||
mkdir -p /private/argus/log/elasticsearch
|
||||
|
||||
# 创建软链接到Elasticsearch预期的数据目录
|
||||
if [ -L /usr/share/elasticsearch/data ]; then
|
||||
rm /usr/share/elasticsearch/data
|
||||
elif [ -d /usr/share/elasticsearch/data ]; then
|
||||
rm -rf /usr/share/elasticsearch/data
|
||||
fi
|
||||
|
||||
ln -sf /private/argus/log/elasticsearch /usr/share/elasticsearch/data
|
||||
|
||||
# 记录容器ip地址
|
||||
DOMAIN=es.log.argus.com
|
||||
IP=`ifconfig | grep -A 1 eth0 | grep inet | awk '{print $2}'`
|
||||
echo current IP: ${IP}
|
||||
echo ${IP} > /private/argus/etc/${DOMAIN}
|
||||
|
||||
echo "[INFO] Data directory linked: /usr/share/elasticsearch/data -> /private/argus/log/elasticsearch"
|
||||
|
||||
# 设置环境变量（ES配置通过docker-compose传递）
|
||||
export ES_JAVA_OPTS="${ES_JAVA_OPTS:-"-Xms512m -Xmx512m"}"
|
||||
|
||||
echo "[INFO] Starting Elasticsearch process..."
|
||||
|
||||
# 启动原始的Elasticsearch entrypoint
|
||||
exec /usr/local/bin/docker-entrypoint.sh elasticsearch
|
||||
39
src/log/elasticsearch/build/supervisord.conf
Normal file
39
src/log/elasticsearch/build/supervisord.conf
Normal file
@@ -0,0 +1,39 @@
|
||||
[supervisord]
|
||||
nodaemon=true
|
||||
logfile=/var/log/supervisor/supervisord.log
|
||||
pidfile=/var/run/supervisord.pid
|
||||
user=root
|
||||
|
||||
[program:elasticsearch]
|
||||
command=/usr/local/bin/start-es-supervised.sh
|
||||
user=elasticsearch
|
||||
stdout_logfile=/var/log/supervisor/elasticsearch.log
|
||||
stderr_logfile=/var/log/supervisor/elasticsearch_error.log
|
||||
autorestart=true
|
||||
startretries=3
|
||||
startsecs=30
|
||||
stopwaitsecs=30
|
||||
killasgroup=true
|
||||
stopasgroup=true
|
||||
|
||||
[program:dns-monitor]
|
||||
command=/usr/local/bin/dns-monitor.sh
|
||||
user=root
|
||||
stdout_logfile=/var/log/supervisor/dns-monitor.log
|
||||
stderr_logfile=/var/log/supervisor/dns-monitor_error.log
|
||||
autorestart=true
|
||||
startretries=3
|
||||
startsecs=5
|
||||
stopwaitsecs=10
|
||||
killasgroup=true
|
||||
stopasgroup=true
|
||||
|
||||
[unix_http_server]
|
||||
file=/var/run/supervisor.sock
|
||||
chmod=0700
|
||||
|
||||
[supervisorctl]
|
||||
serverurl=unix:///var/run/supervisor.sock
|
||||
|
||||
[rpcinterface:supervisor]
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
37
src/log/fluent-bit/build/etc/fluent-bit.conf
Normal file
37
src/log/fluent-bit/build/etc/fluent-bit.conf
Normal file
@@ -0,0 +1,37 @@
|
||||
[SERVICE]
|
||||
Daemon Off
|
||||
Parsers_File parsers.conf
|
||||
HTTP_Server On
|
||||
HTTP_Listen 0.0.0.0
|
||||
HTTP_Port 2020
|
||||
storage.path /buffers
|
||||
storage.sync normal
|
||||
storage.checksum on
|
||||
storage.backlog.mem_limit 128M
|
||||
# 备注：该镜像默认未开启 Hot Reload，修改配置后请重启容器。
|
||||
|
||||
@INCLUDE inputs.d/*.conf
|
||||
|
||||
[FILTER]
|
||||
Name parser
|
||||
Match app.*
|
||||
Key_Name log
|
||||
Parser timestamp_parser
|
||||
Reserve_Data On
|
||||
Preserve_Key On
|
||||
Unescape_Key On
|
||||
|
||||
[FILTER]
|
||||
Name record_modifier
|
||||
Match *
|
||||
Record cluster ${CLUSTER}
|
||||
Record rack ${RACK}
|
||||
Record host ${HOSTNAME}
|
||||
|
||||
[FILTER]
|
||||
Name lua
|
||||
Match app.*
|
||||
script inject_labels.lua
|
||||
call add_labels
|
||||
|
||||
@INCLUDE outputs.d/*.conf
|
||||
15
src/log/fluent-bit/build/etc/inject_labels.lua
Normal file
15
src/log/fluent-bit/build/etc/inject_labels.lua
Normal file
@@ -0,0 +1,15 @@
|
||||
function add_labels(tag, ts, record)
|
||||
record["job_id"] = os.getenv("FB_JOB_ID") or record["job_id"] or "unknown"
|
||||
record["user"] = os.getenv("FB_USER") or record["user"] or "unknown"
|
||||
record["model"] = os.getenv("FB_MODEL") or record["model"] or "unknown"
|
||||
record["gpu_id"] = os.getenv("FB_GPU_ID") or record["gpu_id"] or "na"
|
||||
local p = record["log_path"] or ""
|
||||
if string.find(p, "/logs/infer/") then
|
||||
record["role"] = "infer"
|
||||
elseif string.find(p, "/logs/train/") then
|
||||
record["role"] = "train"
|
||||
else
|
||||
record["role"] = record["role"] or "app"
|
||||
end
|
||||
return 1, ts, record
|
||||
end
|
||||
10
src/log/fluent-bit/build/etc/inputs.d/10-train.conf
Normal file
10
src/log/fluent-bit/build/etc/inputs.d/10-train.conf
Normal file
@@ -0,0 +1,10 @@
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /logs/train/*.log
|
||||
Tag app.train
|
||||
Path_Key log_path
|
||||
Refresh_Interval 5
|
||||
DB /buffers/train.db
|
||||
Skip_Long_Lines On
|
||||
storage.type filesystem
|
||||
multiline.parser python,go,java
|
||||
10
src/log/fluent-bit/build/etc/inputs.d/20-infer.conf
Normal file
10
src/log/fluent-bit/build/etc/inputs.d/20-infer.conf
Normal file
@@ -0,0 +1,10 @@
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /logs/infer/*.log
|
||||
Tag app.infer
|
||||
Path_Key log_path
|
||||
Refresh_Interval 5
|
||||
DB /buffers/infer.db
|
||||
Skip_Long_Lines On
|
||||
storage.type filesystem
|
||||
multiline.parser python,go,java
|
||||
24
src/log/fluent-bit/build/etc/outputs.d/10-es.conf
Normal file
24
src/log/fluent-bit/build/etc/outputs.d/10-es.conf
Normal file
@@ -0,0 +1,24 @@
|
||||
# 重要：使用 Logstash_Format + Logstash_Prefix，生成 train-*/infer-* 索引
|
||||
[OUTPUT]
|
||||
Name es
|
||||
Match app.train
|
||||
Host ${ES_HOST}
|
||||
Port ${ES_PORT}
|
||||
Logstash_Format On
|
||||
Logstash_Prefix train
|
||||
Replace_Dots On
|
||||
Generate_ID On
|
||||
Retry_Limit False
|
||||
Suppress_Type_Name On
|
||||
|
||||
[OUTPUT]
|
||||
Name es
|
||||
Match app.infer
|
||||
Host ${ES_HOST}
|
||||
Port ${ES_PORT}
|
||||
Logstash_Format On
|
||||
Logstash_Prefix infer
|
||||
Replace_Dots On
|
||||
Generate_ID On
|
||||
Retry_Limit False
|
||||
Suppress_Type_Name On
|
||||
28
src/log/fluent-bit/build/etc/parsers.conf
Normal file
28
src/log/fluent-bit/build/etc/parsers.conf
Normal file
@@ -0,0 +1,28 @@
|
||||
[MULTILINE_PARSER]
|
||||
Name python
|
||||
Type regex
|
||||
Flush 2
|
||||
Rule "start_state" "/^\d{4}-\d{2}-\d{2}[\sT]/" "cont"
|
||||
Rule "cont" "/^\s+|^Traceback|^\tat\s+/" "cont"
|
||||
|
||||
[MULTILINE_PARSER]
|
||||
Name go
|
||||
Type regex
|
||||
Flush 2
|
||||
Rule "start_state" "/^[0-9]{4}\/[0-9]{2}\/[0-9]{2}/" "cont"
|
||||
Rule "cont" "/^\s+|^\t/" "cont"
|
||||
|
||||
[MULTILINE_PARSER]
|
||||
Name java
|
||||
Type regex
|
||||
Flush 2
|
||||
Rule "start_state" "/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/" "cont"
|
||||
Rule "cont" "/^\s+at\s+|^\t.../" "cont"
|
||||
|
||||
[PARSER]
|
||||
Name timestamp_parser
|
||||
Format regex
|
||||
Regex ^(?<timestamp>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:Z|[+-]\d{2}:?\d{2}))\s+(?<level>\w+)\s+(?<message>.*)$
|
||||
Time_Key timestamp
|
||||
Time_Format %Y-%m-%dT%H:%M:%S%z
|
||||
Time_Keep On
|
||||
BIN
src/log/fluent-bit/build/packages/fluent-bit_3.1.9_amd64.deb
Normal file
BIN
src/log/fluent-bit/build/packages/fluent-bit_3.1.9_amd64.deb
Normal file
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
109
src/log/fluent-bit/build/start-fluent-bit.sh
Executable file
109
src/log/fluent-bit/build/start-fluent-bit.sh
Executable file
@@ -0,0 +1,109 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "[INFO] Starting Fluent Bit setup in Ubuntu container (offline-first)..."
|
||||
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
# Stage bundle to /tmp (read-only mount under /private)
|
||||
echo "[INFO] Staging fluent-bit bundle..."
|
||||
rm -rf /tmp/flb && mkdir -p /tmp/flb
|
||||
cp -r /private/etc /tmp/flb/
|
||||
mkdir -p /tmp/flb/packages
|
||||
cp -r /private/packages/* /tmp/flb/packages/ 2>/dev/null || true
|
||||
|
||||
# Helper: check and install a local deb if not already satisfied
|
||||
ensure_lib() {
|
||||
local soname="$1"; shift
|
||||
local pattern="$1"; shift
|
||||
if ldconfig -p 2>/dev/null | grep -q "$soname"; then
|
||||
echo "[OK] $soname already present"
|
||||
return 0
|
||||
fi
|
||||
local deb="$(ls /tmp/flb/packages/$pattern 2>/dev/null | head -n1 || true)"
|
||||
if [[ -n "$deb" ]]; then
|
||||
echo "[INFO] Installing local dependency: $(basename "$deb")"
|
||||
dpkg -i "$deb" >/dev/null 2>&1 || true
|
||||
else
|
||||
echo "[WARN] Local deb for $soname not found (pattern=$pattern)"
|
||||
fi
|
||||
if ! ldconfig -p 2>/dev/null | grep -q "$soname"; then
|
||||
echo "[WARN] $soname still missing after local install; attempting apt fallback"
|
||||
apt-get update -qq || true
|
||||
case "$soname" in
|
||||
libpq.so.5) apt-get install -y -qq libpq5 || true ;;
|
||||
libyaml-0.so.2) apt-get install -y -qq libyaml-0-2 || true ;;
|
||||
esac
|
||||
fi
|
||||
ldconfig 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Offline-first: satisfy runtime deps from local debs, fallback to apt only if necessary
|
||||
ensure_lib "libpq.so.5" "libpq5_*_amd64.deb"
|
||||
ensure_lib "libyaml-0.so.2" "libyaml-0-2_*_amd64.deb"
|
||||
ensure_lib "libsasl2.so.2" "libsasl2-2_*_amd64.deb"
|
||||
ensure_lib "libldap-2.5.so.0" "libldap-2.5-0_*_amd64.deb"
|
||||
|
||||
# Install fluent-bit main package from local bundle
|
||||
FLB_DEB="$(ls /tmp/flb/packages/fluent-bit_*_amd64.deb 2>/dev/null | head -n1 || true)"
|
||||
if [[ -z "$FLB_DEB" ]]; then
|
||||
echo "[ERROR] fluent-bit deb not found under /private/packages" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "[INFO] Installing Fluent Bit: $(basename "$FLB_DEB")"
|
||||
dpkg -i "$FLB_DEB" >/dev/null 2>&1 || true
|
||||
|
||||
# If dpkg reported unresolved dependencies, try apt -f only as last resort
|
||||
if ! command -v /opt/fluent-bit/bin/fluent-bit >/dev/null 2>&1; then
|
||||
echo "[WARN] fluent-bit binary missing after dpkg; attempting apt --fix-broken"
|
||||
apt-get install -f -y -qq || true
|
||||
fi
|
||||
|
||||
# Ensure runtime library dependencies are satisfied (libsasl2, libldap are required via libpq/curl)
|
||||
MISSING=$(ldd /opt/fluent-bit/bin/fluent-bit 2>/dev/null | awk '/not found/{print $1}' | xargs -r echo || true)
|
||||
if [[ -n "$MISSING" ]]; then
|
||||
echo "[WARN] missing shared libs: $MISSING"
|
||||
apt-get update -qq || true
|
||||
apt-get install -y -qq libsasl2-2 libldap-2.5-0 || true
|
||||
apt-get install -f -y -qq || true
|
||||
fi
|
||||
|
||||
echo "[INFO] Fluent Bit version:"
|
||||
/opt/fluent-bit/bin/fluent-bit --version || { echo "[ERROR] fluent-bit not installed or libraries missing" >&2; exit 1; }
|
||||
|
||||
# Place configuration
|
||||
mkdir -p /etc/fluent-bit
|
||||
cp -r /tmp/flb/etc/* /etc/fluent-bit/
|
||||
|
||||
# Create logs/buffers dirs
|
||||
mkdir -p /logs/train /logs/infer /buffers
|
||||
|
||||
# 控制日志目录权限：默认对宿主 bind mount 目录采用 1777（可由环境变量关闭）
|
||||
: "${ARGUS_LOGS_WORLD_WRITABLE:=1}"
|
||||
if [[ "${ARGUS_LOGS_WORLD_WRITABLE}" == "1" ]]; then
|
||||
chmod 1777 /logs/train /logs/infer || true
|
||||
else
|
||||
chmod 755 /logs/train /logs/infer || true
|
||||
fi
|
||||
|
||||
# 缓冲目录仅供进程使用，不对外开放写入
|
||||
chmod 770 /buffers || true
|
||||
|
||||
# 目录属主设置为 fluent-bit（不影响 1777 粘滞位）
|
||||
chown -R fluent-bit:fluent-bit /logs /buffers 2>/dev/null || true
|
||||
|
||||
# Wait for Elasticsearch via bash /dev/tcp to avoid curl dependency
|
||||
echo "[INFO] Waiting for Elasticsearch to be ready (tcp ${ES_HOST}:${ES_PORT})..."
|
||||
for i in $(seq 1 120); do
|
||||
if exec 3<>/dev/tcp/${ES_HOST}/${ES_PORT}; then
|
||||
exec 3<&- 3>&-
|
||||
echo "[INFO] Elasticsearch is ready"
|
||||
break
|
||||
fi
|
||||
[[ $i -eq 120 ]] && { echo "[ERROR] ES not reachable" >&2; exit 1; }
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "[INFO] Starting Fluent Bit with configuration from /etc/fluent-bit/"
|
||||
echo "[INFO] Command: /opt/fluent-bit/bin/fluent-bit --config=/etc/fluent-bit/fluent-bit.conf"
|
||||
exec /opt/fluent-bit/bin/fluent-bit --config=/etc/fluent-bit/fluent-bit.conf
|
||||
79
src/log/kibana/build/Dockerfile
Normal file
79
src/log/kibana/build/Dockerfile
Normal file
@@ -0,0 +1,79 @@
|
||||
FROM docker.elastic.co/kibana/kibana:8.13.4
|
||||
|
||||
# 切换到 root 用户进行系统级安装
|
||||
USER root
|
||||
|
||||
ARG ARGUS_BUILD_UID=2133
|
||||
ARG ARGUS_BUILD_GID=2015
|
||||
|
||||
ENV ARGUS_BUILD_UID=${ARGUS_BUILD_UID} \
|
||||
ARGUS_BUILD_GID=${ARGUS_BUILD_GID}
|
||||
|
||||
# 调整 kibana 用户与用户组 ID 以匹配宿主机配置
|
||||
RUN set -eux; \
|
||||
current_gid="$(getent group kibana | awk -F: '{print $3}')"; \
|
||||
if [ -z "$current_gid" ]; then \
|
||||
groupadd -g "${ARGUS_BUILD_GID}" kibana; \
|
||||
elif [ "$current_gid" != "${ARGUS_BUILD_GID}" ]; then \
|
||||
groupmod -g "${ARGUS_BUILD_GID}" kibana; \
|
||||
fi; \
|
||||
if id kibana >/dev/null 2>&1; then \
|
||||
current_uid="$(id -u kibana)"; \
|
||||
if [ "$current_uid" != "${ARGUS_BUILD_UID}" ]; then \
|
||||
usermod -u "${ARGUS_BUILD_UID}" kibana; \
|
||||
fi; \
|
||||
else \
|
||||
useradd -m -u "${ARGUS_BUILD_UID}" -g "${ARGUS_BUILD_GID}" kibana; \
|
||||
fi; \
|
||||
chown -R "${ARGUS_BUILD_UID}:${ARGUS_BUILD_GID}" /usr/share/kibana
|
||||
|
||||
# 设置构建参数
|
||||
ARG USE_INTRANET=false
|
||||
|
||||
# 配置内网 apt 源 (如果指定了内网选项)
|
||||
RUN if [ "$USE_INTRANET" = "true" ]; then \
|
||||
echo "Configuring intranet apt sources..." && \
|
||||
cp /etc/apt/sources.list /etc/apt/sources.list.bak && \
|
||||
echo "deb [trusted=yes] http://10.68.64.1/ubuntu2204/ jammy main" > /etc/apt/sources.list && \
|
||||
echo 'Acquire::https::Verify-Peer "false";' > /etc/apt/apt.conf.d/99disable-ssl-check && \
|
||||
echo 'Acquire::https::Verify-Host "false";' >> /etc/apt/apt.conf.d/99disable-ssl-check; \
|
||||
fi
|
||||
|
||||
# 安装 supervisor, net-tools, vim
|
||||
RUN apt-get update && \
|
||||
apt-get install -y supervisor net-tools inetutils-ping vim && \
|
||||
apt-get clean && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# 配置部署时使用的apt源
|
||||
RUN if [ "$USE_INTRANET" = "true" ]; then \
|
||||
echo "deb [trusted=yes] https://10.92.132.52/mirrors/ubuntu2204/ jammy main" > /etc/apt/sources.list; \
|
||||
fi
|
||||
|
||||
# 创建 supervisor 日志目录
|
||||
RUN mkdir -p /var/log/supervisor
|
||||
|
||||
|
||||
# 复制 supervisor 配置文件
|
||||
COPY src/log/kibana/build/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
|
||||
|
||||
# 复制启动脚本
|
||||
COPY src/log/kibana/build/start-kibana-supervised.sh /usr/local/bin/start-kibana-supervised.sh
|
||||
COPY src/log/kibana/build/kibana-post-start.sh /usr/local/bin/kibana-post-start.sh
|
||||
RUN chmod +x /usr/local/bin/start-kibana-supervised.sh /usr/local/bin/kibana-post-start.sh
|
||||
|
||||
# 复制DNS监控脚本
|
||||
COPY src/log/kibana/build/dns-monitor.sh /usr/local/bin/dns-monitor.sh
|
||||
RUN chmod +x /usr/local/bin/dns-monitor.sh
|
||||
|
||||
# kibana需要用到 /root/.config/puppeteer 路径
|
||||
RUN chmod 777 /root
|
||||
|
||||
# 保持 root 用户，由 supervisor 管理用户切换
|
||||
USER root
|
||||
|
||||
# 暴露端口
|
||||
EXPOSE 5601
|
||||
|
||||
# 使用 supervisor 作为入口点
|
||||
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
|
||||
1
src/log/kibana/build/dns-monitor.sh
Symbolic link
1
src/log/kibana/build/dns-monitor.sh
Symbolic link
@@ -0,0 +1 @@
|
||||
../../../bind/build/dns-monitor.sh
|
||||
133
src/log/kibana/build/kibana-post-start.sh
Normal file
133
src/log/kibana/build/kibana-post-start.sh
Normal file
@@ -0,0 +1,133 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
ES_HOST="${ELASTICSEARCH_HOSTS:-http://es:9200}"
|
||||
KB_HOST="${KB_HOST:-http://127.0.0.1:5601}"
|
||||
|
||||
echo "[INFO] Starting Kibana post-start configuration..."
|
||||
|
||||
# 等待 Elasticsearch 可用
|
||||
wait_for_elasticsearch() {
|
||||
echo "[INFO] Waiting for Elasticsearch..."
|
||||
local max_attempts=60
|
||||
local attempt=1
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if curl -fs "$ES_HOST/_cluster/health" >/dev/null 2>&1; then
|
||||
echo "[OK] Elasticsearch is available"
|
||||
return 0
|
||||
fi
|
||||
echo " Waiting for ES... ($attempt/$max_attempts)"
|
||||
sleep 5
|
||||
((attempt++))
|
||||
done
|
||||
|
||||
echo "[ERROR] Elasticsearch timeout"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 等待 Kibana 可用
|
||||
wait_for_kibana() {
|
||||
echo "[INFO] Waiting for Kibana..."
|
||||
local max_attempts=120
|
||||
local attempt=1
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if curl -fs "$KB_HOST/api/status" >/dev/null 2>&1; then
|
||||
local status=$(curl -s "$KB_HOST/api/status" | grep -o '"level":"available"' || echo "")
|
||||
if [ -n "$status" ]; then
|
||||
echo "[OK] Kibana is available"
|
||||
return 0
|
||||
fi
|
||||
echo " Waiting for Kibana... ($attempt/$max_attempts, status: $status)"
|
||||
else
|
||||
echo " Waiting for Kibana... ($attempt/$max_attempts, connection failed)"
|
||||
fi
|
||||
sleep 5
|
||||
((attempt++))
|
||||
done
|
||||
|
||||
echo "[ERROR] Kibana timeout"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 幂等设置索引副本数为0
|
||||
fix_replicas_idempotent() {
|
||||
echo "[INFO] Checking and fixing index replicas..."
|
||||
|
||||
# 获取所有 train-* 和 infer-* 索引
|
||||
local indices=$(curl -s "$ES_HOST/_cat/indices/train-*,infer-*?h=index" 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$indices" ]; then
|
||||
echo "[INFO] No train-*/infer-* indices found, skipping replica adjustment"
|
||||
return 0
|
||||
fi
|
||||
|
||||
for idx in $indices; do
|
||||
# 检查当前副本数
|
||||
local current_replicas=$(curl -s "$ES_HOST/$idx/_settings" | grep -o '"number_of_replicas":"[^"]*"' | cut -d'"' -f4 || echo "")
|
||||
|
||||
if [ "$current_replicas" != "0" ]; then
|
||||
echo "[INFO] Setting replicas to 0 for index: $idx (current: $current_replicas)"
|
||||
curl -fsS -X PUT "$ES_HOST/$idx/_settings" \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"index":{"number_of_replicas":0}}' >/dev/null || {
|
||||
echo "[WARN] Failed to set replicas for $idx"
|
||||
continue
|
||||
}
|
||||
echo "[OK] Updated replicas for $idx"
|
||||
else
|
||||
echo "[INFO] Index $idx already has 0 replicas, skipping"
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
# 幂等创建数据视图
|
||||
create_or_ensure_data_view() {
|
||||
local name="$1"
|
||||
local title="$2"
|
||||
|
||||
local list_response
|
||||
list_response=$(curl -fsS "$KB_HOST/api/data_views" -H 'kbn-xsrf: true' 2>/dev/null || echo "")
|
||||
|
||||
if [ -z "$list_response" ]; then
|
||||
echo "[WARN] Failed to list data views, skipping creation check for $title"
|
||||
return
|
||||
fi
|
||||
|
||||
if echo "$list_response" | grep -Fq "\"title\":\"$title\""; then
|
||||
echo "[INFO] Data view $title already exists, skipping"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "[INFO] Creating data view for $title indices (allowNoIndex)"
|
||||
|
||||
curl -fsS -X POST "$KB_HOST/api/data_views/data_view?allowNoIndex=true" \
|
||||
-H 'kbn-xsrf: true' \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d "{\"data_view\":{\"name\":\"$name\",\"title\":\"$title\",\"timeFieldName\":\"@timestamp\",\"allowNoIndex\":true}}" \
|
||||
>/dev/null && echo "[OK] Created $name data view" || echo "[WARN] Failed to create $name data view"
|
||||
}
|
||||
|
||||
create_data_views_idempotent() {
|
||||
echo "[INFO] Checking and creating data views..."
|
||||
|
||||
create_or_ensure_data_view "train" "train-*"
|
||||
create_or_ensure_data_view "infer" "infer-*"
|
||||
}
|
||||
|
||||
# 主逻辑
|
||||
main() {
|
||||
# 等待服务可用
|
||||
wait_for_elasticsearch || exit 1
|
||||
wait_for_kibana || exit 1
|
||||
|
||||
# 执行幂等配置
|
||||
fix_replicas_idempotent
|
||||
create_data_views_idempotent
|
||||
|
||||
echo "[INFO] Kibana post-start configuration completed"
|
||||
}
|
||||
|
||||
# 运行主逻辑
|
||||
main
|
||||
37
src/log/kibana/build/start-kibana-supervised.sh
Normal file
37
src/log/kibana/build/start-kibana-supervised.sh
Normal file
@@ -0,0 +1,37 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "[INFO] Starting Kibana under supervisor..."
|
||||
|
||||
mkdir -p /private/argus/log/kibana
|
||||
|
||||
# 创建软链接到Kibana预期的数据目录
|
||||
if [ -L /usr/share/kibana/data ]; then
|
||||
rm /usr/share/kibana/data
|
||||
elif [ -d /usr/share/kibana/data ]; then
|
||||
rm -rf /usr/share/kibana/data
|
||||
fi
|
||||
|
||||
ln -sf /private/argus/log/kibana /usr/share/kibana/data
|
||||
|
||||
echo "[INFO] Data directory linked: /usr/share/kibana/data -> /private/argus/log/kibana"
|
||||
|
||||
# 记录容器ip地址
|
||||
DOMAIN=kibana.log.argus.com
|
||||
IP=`ifconfig | grep -A 1 eth0 | grep inet | awk '{print $2}'`
|
||||
echo current IP: ${IP}
|
||||
echo ${IP} > /private/argus/etc/${DOMAIN}
|
||||
|
||||
# 设置环境变量
|
||||
export ELASTICSEARCH_HOSTS="${ELASTICSEARCH_HOSTS:-"http://es:9200"}"
|
||||
|
||||
echo "[INFO] Connecting to Elasticsearch at: $ELASTICSEARCH_HOSTS"
|
||||
|
||||
# 启动后台配置任务
|
||||
echo "[INFO] Starting background post-start configuration..."
|
||||
/usr/local/bin/kibana-post-start.sh &
|
||||
|
||||
echo "[INFO] Starting Kibana process..."
|
||||
|
||||
# 启动原始的Kibana entrypoint
|
||||
exec /usr/local/bin/kibana-docker
|
||||
39
src/log/kibana/build/supervisord.conf
Normal file
39
src/log/kibana/build/supervisord.conf
Normal file
@@ -0,0 +1,39 @@
|
||||
[supervisord]
|
||||
nodaemon=true
|
||||
logfile=/var/log/supervisor/supervisord.log
|
||||
pidfile=/var/run/supervisord.pid
|
||||
user=root
|
||||
|
||||
[program:kibana]
|
||||
command=/usr/local/bin/start-kibana-supervised.sh
|
||||
user=kibana
|
||||
stdout_logfile=/var/log/supervisor/kibana.log
|
||||
stderr_logfile=/var/log/supervisor/kibana_error.log
|
||||
autorestart=true
|
||||
startretries=3
|
||||
startsecs=30
|
||||
stopwaitsecs=30
|
||||
killasgroup=true
|
||||
stopasgroup=true
|
||||
|
||||
[program:dns-monitor]
|
||||
command=/usr/local/bin/dns-monitor.sh
|
||||
user=root
|
||||
stdout_logfile=/var/log/supervisor/dns-monitor.log
|
||||
stderr_logfile=/var/log/supervisor/dns-monitor_error.log
|
||||
autorestart=true
|
||||
startretries=3
|
||||
startsecs=5
|
||||
stopwaitsecs=10
|
||||
killasgroup=true
|
||||
stopasgroup=true
|
||||
|
||||
[unix_http_server]
|
||||
file=/var/run/supervisor.sock
|
||||
chmod=0700
|
||||
|
||||
[supervisorctl]
|
||||
serverurl=unix:///var/run/supervisor.sock
|
||||
|
||||
[rpcinterface:supervisor]
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
84
src/log/tests/docker-compose.yml
Normal file
84
src/log/tests/docker-compose.yml
Normal file
@@ -0,0 +1,84 @@
|
||||
version: "3.8"
|
||||
services:
|
||||
es:
|
||||
build:
|
||||
context: ../elasticsearch/build
|
||||
dockerfile: Dockerfile
|
||||
image: argus-elasticsearch:latest
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- xpack.security.enabled=false
|
||||
- ES_JAVA_OPTS=-Xms512m -Xmx512m
|
||||
volumes:
|
||||
- ./private/argus/:/private/argus/
|
||||
ports: ["9200:9200"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fs http://localhost:9200 >/dev/null || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
restart: always
|
||||
|
||||
kibana:
|
||||
build:
|
||||
context: ../kibana/build
|
||||
dockerfile: Dockerfile
|
||||
image: argus-kibana:latest
|
||||
environment:
|
||||
- ELASTICSEARCH_HOSTS=http://es.log.argus.com:9200
|
||||
volumes:
|
||||
- ./private/argus/:/private/argus/
|
||||
ports: ["5601:5601"]
|
||||
depends_on:
|
||||
es:
|
||||
condition: service_healthy
|
||||
|
||||
fluent-bit-host01:
|
||||
image: ubuntu:22.04
|
||||
environment:
|
||||
- CLUSTER=local
|
||||
- RACK=dev
|
||||
- HOSTNAME=host01
|
||||
- ES_HOST=es
|
||||
- ES_PORT=9200
|
||||
volumes:
|
||||
- ../fluent-bit/build:/private/
|
||||
ports: ["2020:2020"]
|
||||
depends_on:
|
||||
es:
|
||||
condition: service_healthy
|
||||
command: /private/start-fluent-bit.sh
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fs http://localhost:2020/api/v2/metrics >/dev/null || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
|
||||
fluent-bit-host02:
|
||||
image: ubuntu:22.04
|
||||
environment:
|
||||
- CLUSTER=local
|
||||
- RACK=dev
|
||||
- HOSTNAME=host02
|
||||
- ES_HOST=es
|
||||
- ES_PORT=9200
|
||||
volumes:
|
||||
- ../fluent-bit/build:/private/
|
||||
ports: ["2021:2020"]
|
||||
depends_on:
|
||||
es:
|
||||
condition: service_healthy
|
||||
command: /private/start-fluent-bit.sh
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fs http://localhost:2020/api/v2/metrics >/dev/null || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
restart: always
|
||||
|
||||
bind9:
|
||||
image: argus-bind9:latest
|
||||
volumes:
|
||||
- ./private/argus:/private/argus/
|
||||
restart: always
|
||||
|
||||
73
src/log/tests/scripts/01_bootstrap.sh
Executable file
73
src/log/tests/scripts/01_bootstrap.sh
Executable file
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd)"
|
||||
project_root="$(cd "$root/../../.." && pwd)"
|
||||
|
||||
source "$project_root/scripts/common/build_user.sh"
|
||||
load_build_user
|
||||
|
||||
# 创建新的private目录结构 (基于argus目录结构)
|
||||
echo "[INFO] Creating private directory structure for supervisor-based containers..."
|
||||
mkdir -p "$root/private/argus/log/elasticsearch"
|
||||
mkdir -p "$root/private/argus/log/kibana"
|
||||
mkdir -p "$root/private/argus/etc/"
|
||||
|
||||
|
||||
# 设置数据目录权限（ES 和 Kibana 容器都使用 UID 1000）
|
||||
echo "[INFO] Setting permissions for data directories..."
|
||||
chown -R "${ARGUS_BUILD_UID}:${ARGUS_BUILD_GID}" "$root/private/argus/log/elasticsearch" 2>/dev/null || true
|
||||
chown -R "${ARGUS_BUILD_UID}:${ARGUS_BUILD_GID}" "$root/private/argus/log/kibana" 2>/dev/null || true
|
||||
chown -R "${ARGUS_BUILD_UID}:${ARGUS_BUILD_GID}" "$root/private/argus/etc" 2>/dev/null || true
|
||||
|
||||
echo "[INFO] Supervisor-based containers will manage their own scripts and configurations"
|
||||
|
||||
# 检查fluent-bit相关文件是否存在
|
||||
if [[ ! -f "$root/../fluent-bit/fluent-bit-bundle.tar.gz" ]]; then
|
||||
echo "[WARN] fluent-bit/fluent-bit-bundle.tar.gz 不存在，请确保已创建该文件"
|
||||
fi
|
||||
|
||||
if [[ ! -f "$root/../fluent-bit/start-fluent-bit.sh" ]]; then
|
||||
echo "[WARN] fluent-bit/start-fluent-bit.sh 不存在，请确保已创建该启动脚本"
|
||||
fi
|
||||
|
||||
echo "[OK] 初始化完成: private/argus/log/{elasticsearch,kibana}"
|
||||
echo "[INFO] Fluent-bit files should be in fluent-bit/ directory"
|
||||
|
||||
# 准备 Fluent Bit 离线依赖（从 metric all-in-one-full 复制 deb 到 ../fluent-bit/build/packages）
|
||||
FLB_BUILD_PACKAGES_DIR="$root/../fluent-bit/build/packages"
|
||||
mkdir -p "$FLB_BUILD_PACKAGES_DIR"
|
||||
for deb in \
|
||||
"$project_root/src/metric/client-plugins/all-in-one-full/plugins/fluent-bit/bin/libyaml-0-2_"*_amd64.deb \
|
||||
"$project_root/src/metric/client-plugins/all-in-one-full/plugins/fluent-bit/bin/libpq5_"*_amd64.deb ; do
|
||||
if ls $deb >/dev/null 2>&1; then
|
||||
for f in $deb; do
|
||||
base="$(basename "$f")"
|
||||
if [[ ! -f "$FLB_BUILD_PACKAGES_DIR/$base" ]]; then
|
||||
cp "$f" "$FLB_BUILD_PACKAGES_DIR/"
|
||||
echo " [+] copied $base"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
done
|
||||
|
||||
# 额外：从 all-in-one-full 的 ubuntu22/curl.tar.gz 解包必要依赖（libsasl2/ldap），便于离线安装
|
||||
CURLOPT_TAR="$project_root/src/metric/client-plugins/all-in-one-full/deps/ubuntu22/curl.tar.gz"
|
||||
if [[ -f "$CURLOPT_TAR" ]]; then
|
||||
tmpdir=$(mktemp -d)
|
||||
if tar -xzf "$CURLOPT_TAR" -C "$tmpdir" 2>/dev/null; then
|
||||
for p in \
|
||||
libsasl2-2_*_amd64.deb \
|
||||
libsasl2-modules-db_*_amd64.deb \
|
||||
libldap-2.5-0_*_amd64.deb \
|
||||
libidn2-0_*_amd64.deb \
|
||||
libbrotli1_*_amd64.deb \
|
||||
libssl3_*_amd64.deb ; do
|
||||
src=$(ls "$tmpdir"/curl/$p 2>/dev/null | head -n1 || true)
|
||||
if [[ -n "$src" ]]; then
|
||||
base="$(basename "$src")"
|
||||
[[ -f "$FLB_BUILD_PACKAGES_DIR/$base" ]] || cp "$src" "$FLB_BUILD_PACKAGES_DIR/" && echo " [+] staged $base"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
rm -rf "$tmpdir"
|
||||
fi
|
||||
10
src/log/tests/scripts/02_up.sh
Executable file
10
src/log/tests/scripts/02_up.sh
Executable file
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
compose_cmd="docker compose"
|
||||
if ! $compose_cmd version >/dev/null 2>&1; then
|
||||
if command -v docker-compose >/dev/null 2>&1; then compose_cmd="docker-compose"; else
|
||||
echo "需要 Docker Compose，请安装后重试" >&2; exit 1; fi
|
||||
fi
|
||||
$compose_cmd -p logging-mvp up -d --remove-orphans
|
||||
echo "[OK] 服务已启动：ES http://localhost:9200 Kibana http://localhost:5601 Fluent-Bit host01 http://localhost:2020 Fluent-Bit host02 http://localhost:2021"
|
||||
45
src/log/tests/scripts/03_send_test_host01.sh
Executable file
45
src/log/tests/scripts/03_send_test_host01.sh
Executable file
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# 获取fluent-bit-host01容器名称
|
||||
container_name="logging-mvp-fluent-bit-host01-1"
|
||||
|
||||
wait_for_container() {
|
||||
local name="$1"
|
||||
local attempts=30
|
||||
local delay=5
|
||||
local i
|
||||
for ((i = 1; i <= attempts; i++)); do
|
||||
if docker ps --format '{{.Names}}' | grep -Fx "$name" >/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
echo "[INFO] 等待容器 $name 启动中... ($i/$attempts)"
|
||||
sleep "$delay"
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
if ! wait_for_container "$container_name"; then
|
||||
echo "[ERROR] Fluent Bit容器 $container_name 未运行"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 创建日志目录
|
||||
docker exec "$container_name" mkdir -p /logs/train /logs/infer
|
||||
|
||||
# 写入训练日志 (host01)
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host01] training step=1 loss=1.23 model=bert\n' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" >> /logs/train/train-demo.log"
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host01] training step=2 loss=1.15 model=bert\n' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" >> /logs/train/train-demo.log"
|
||||
|
||||
# 写入推理日志 (host01)
|
||||
docker exec "$container_name" sh -c "printf '%s ERROR [host01] inference failed on batch=1\n' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" >> /logs/infer/infer-demo.log"
|
||||
docker exec "$container_name" sh -c "cat <<'STACK' >> /logs/infer/infer-demo.log
|
||||
Traceback (most recent call last):
|
||||
File \"inference.py\", line 15, in <module>
|
||||
raise RuntimeError(\"CUDA out of memory on host01\")
|
||||
RuntimeError: CUDA out of memory on host01
|
||||
STACK"
|
||||
|
||||
echo "[OK] 已通过docker exec写入测试日志到 host01 容器内："
|
||||
echo " - /logs/train/train-demo.log"
|
||||
echo " - /logs/infer/infer-demo.log"
|
||||
41
src/log/tests/scripts/03_send_test_host02.sh
Executable file
41
src/log/tests/scripts/03_send_test_host02.sh
Executable file
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# 获取fluent-bit-host02容器名称
|
||||
container_name="logging-mvp-fluent-bit-host02-1"
|
||||
|
||||
wait_for_container() {
|
||||
local name="$1"
|
||||
local attempts=30
|
||||
local delay=5
|
||||
local i
|
||||
for ((i = 1; i <= attempts; i++)); do
|
||||
if docker ps --format '{{.Names}}' | grep -Fx "$name" >/dev/null; then
|
||||
return 0
|
||||
fi
|
||||
echo "[INFO] 等待容器 $name 启动中... ($i/$attempts)"
|
||||
sleep "$delay"
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
if ! wait_for_container "$container_name"; then
|
||||
echo "[ERROR] Fluent Bit容器 $container_name 未运行"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 创建日志目录
|
||||
docker exec "$container_name" mkdir -p /logs/train /logs/infer
|
||||
|
||||
# 写入训练日志 (host02)
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host02] training step=1 loss=1.45 model=gpt\n' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" >> /logs/train/train-demo.log"
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host02] training step=2 loss=1.38 model=gpt\n' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" >> /logs/train/train-demo.log"
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host02] training step=3 loss=1.32 model=gpt\n' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" >> /logs/train/train-demo.log"
|
||||
|
||||
# 写入推理日志 (host02)
|
||||
docker exec "$container_name" sh -c "printf '%s WARN [host02] inference slow on batch=5 latency=2.3s\n' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" >> /logs/infer/infer-demo.log"
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host02] inference completed batch=6 latency=0.8s\n' \"\$(date -u +%Y-%m-%dT%H:%M:%SZ)\" >> /logs/infer/infer-demo.log"
|
||||
|
||||
echo "[OK] 已通过docker exec写入测试日志到 host02 容器内："
|
||||
echo " - /logs/train/train-demo.log"
|
||||
echo " - /logs/infer/infer-demo.log"
|
||||
42
src/log/tests/scripts/04_query_es.sh
Executable file
42
src/log/tests/scripts/04_query_es.sh
Executable file
@@ -0,0 +1,42 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# ES endpoint and wait strategy
|
||||
ES="${ES:-http://localhost:9200}"
|
||||
es_wait_attempts="${ES_WAIT_ATTEMPTS:-60}" # total attempts to wait for ES
|
||||
es_wait_interval="${ES_WAIT_INTERVAL:-2}" # seconds between attempts
|
||||
|
||||
echo "[i] 查询 ES 端点：$ES"
|
||||
|
||||
wait_for_es() {
|
||||
local attempt=1
|
||||
while (( attempt <= es_wait_attempts )); do
|
||||
# 等待集群达到至少 yellow 状态；请求失败则重试
|
||||
if curl -fsS "$ES/_cluster/health?wait_for_status=yellow&timeout=1s" >/dev/null 2>&1; then
|
||||
echo "[ok] Elasticsearch 已就绪 (attempt=${attempt}/${es_wait_attempts})"
|
||||
return 0
|
||||
fi
|
||||
echo "[..] 等待 Elasticsearch 可用中 (${attempt}/${es_wait_attempts})"
|
||||
sleep "${es_wait_interval}"
|
||||
(( attempt++ ))
|
||||
done
|
||||
echo "[err] Elasticsearch 在 ${es_wait_attempts} 次尝试后仍不可用"
|
||||
return 1
|
||||
}
|
||||
|
||||
safe_count() {
|
||||
# 对缺失索引返回 0，避免 404 触发失败
|
||||
local pattern="$1"
|
||||
local json
|
||||
json=$(curl -fsS "$ES/${pattern}/_count?ignore_unavailable=true&allow_no_indices=true" 2>/dev/null || echo '{}')
|
||||
echo "$json" | sed -E 's/.*"count":([0-9]+).*/\1/' | awk 'NF{print $0;exit} END{if(NR==0)print 0}'
|
||||
}
|
||||
|
||||
wait_for_es
|
||||
|
||||
# 列出相关索引（可能为空，允许）
|
||||
curl -fsS "$ES/_cat/indices?v" | egrep 'train-|infer-|logstash' || true
|
||||
|
||||
# 打印计数，缺失索引按 0 处理
|
||||
printf "train-* 计数："; safe_count "train-*"; echo
|
||||
printf "infer-* 计数："; safe_count "infer-*"; echo
|
||||
21
src/log/tests/scripts/05_down.sh
Executable file
21
src/log/tests/scripts/05_down.sh
Executable file
@@ -0,0 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
compose_cmd="docker compose"
|
||||
if ! $compose_cmd version >/dev/null 2>&1; then
|
||||
if command -v docker-compose >/dev/null 2>&1; then compose_cmd="docker-compose"; else
|
||||
echo "需要 Docker Compose，请安装后重试" >&2; exit 1; fi
|
||||
fi
|
||||
$compose_cmd -p logging-mvp down
|
||||
echo "[OK] 已停止所有容器"
|
||||
|
||||
# 清理private目录内容
|
||||
echo "[INFO] 清理private目录内容..."
|
||||
cd "$(dirname "$0")/.."
|
||||
if [ -d "private" ]; then
|
||||
# 删除private目录及其所有内容
|
||||
rm -rf private
|
||||
echo "[OK] 已清理private目录"
|
||||
else
|
||||
echo "[INFO] private目录不存在，无需清理"
|
||||
fi
|
||||
208
src/log/tests/scripts/06_dns_test.sh
Executable file
208
src/log/tests/scripts/06_dns_test.sh
Executable file
@@ -0,0 +1,208 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "======================================="
|
||||
echo "ARGUS DNS监控功能测试"
|
||||
echo "======================================="
|
||||
echo ""
|
||||
|
||||
# 记录测试开始时间
|
||||
test_start_time=$(date +%s)
|
||||
|
||||
# 函数：显示测试步骤
|
||||
show_step() {
|
||||
echo ""
|
||||
echo "🔄 Step $1: $2"
|
||||
echo "----------------------------------------"
|
||||
}
|
||||
|
||||
# 函数：验证步骤结果
|
||||
verify_step() {
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ $1 - SUCCESS"
|
||||
else
|
||||
echo "❌ $1 - FAILED"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 函数：等待服务就绪
|
||||
wait_for_services() {
|
||||
echo "[INFO] Waiting for services to be ready..."
|
||||
local max_attempts=60
|
||||
local attempt=1
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if curl -fs http://localhost:9200/_cluster/health >/dev/null 2>&1 && \
|
||||
curl -fs http://localhost:5601/api/status >/dev/null 2>&1; then
|
||||
echo "[OK] Services are ready!"
|
||||
return 0
|
||||
fi
|
||||
echo " Waiting for services... ($attempt/$max_attempts)"
|
||||
sleep 5
|
||||
((attempt++))
|
||||
done
|
||||
|
||||
echo "[ERROR] Services not ready after $max_attempts attempts"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 函数：检查容器中的/etc/resolv.conf
|
||||
check_resolv_conf() {
|
||||
local service_name=$1
|
||||
local expected_dns=$2
|
||||
|
||||
echo "[INFO] 检查 $service_name 容器的 /etc/resolv.conf..."
|
||||
|
||||
local resolv_content=$(docker exec "${service_name}" cat /etc/resolv.conf 2>/dev/null || echo "")
|
||||
if echo "$resolv_content" | grep -q "nameserver $expected_dns"; then
|
||||
echo "✅ $service_name resolv.conf contains nameserver $expected_dns"
|
||||
return 0
|
||||
else
|
||||
echo "❌ $service_name resolv.conf does not contain nameserver $expected_dns"
|
||||
echo "实际内容:"
|
||||
echo "$resolv_content"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 函数：检查DNS监控日志
|
||||
check_dns_monitor_logs() {
|
||||
local service_name=$1
|
||||
|
||||
echo "[INFO] 检查 $service_name 的DNS监控日志..."
|
||||
|
||||
local dns_logs=$(docker exec "$service_name" tail -n 20 /var/log/supervisor/dns-monitor.log 2>/dev/null || echo "")
|
||||
if [ -n "$dns_logs" ]; then
|
||||
echo "✅ $service_name DNS监控日志存在"
|
||||
echo "最近的日志:"
|
||||
echo "$dns_logs"
|
||||
return 0
|
||||
else
|
||||
echo "❌ $service_name DNS监控日志为空或不存在"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 函数：确保目录结构存在
|
||||
ensure_directories() {
|
||||
echo "[INFO] 确保目录结构存在..."
|
||||
# 确保目录存在
|
||||
mkdir -p ./private/argus/etc/
|
||||
echo "✅ 目录结构准备完成（注：使用真实的update-dns.sh脚本）"
|
||||
}
|
||||
|
||||
# 开始DNS监控测试
|
||||
show_step "1" "Bootstrap - Initialize environment"
|
||||
./scripts/01_bootstrap.sh
|
||||
verify_step "Bootstrap"
|
||||
|
||||
# 确保目录结构
|
||||
ensure_directories
|
||||
|
||||
show_step "2" "Startup - Start all services"
|
||||
./scripts/02_up.sh
|
||||
verify_step "Service startup"
|
||||
|
||||
# 等待服务完全就绪
|
||||
wait_for_services || exit 1
|
||||
|
||||
show_step "3" "Create initial DNS configuration"
|
||||
# 创建初始的DNS配置文件 - 只有一个IP
|
||||
echo "[INFO] 创建初始的dns.conf文件 (8.8.8.8)..."
|
||||
cat > ./private/argus/etc/dns.conf << 'EOF'
|
||||
8.8.8.8
|
||||
EOF
|
||||
|
||||
echo "✅ 初始dns.conf文件创建成功 (8.8.8.8)"
|
||||
verify_step "Initial DNS configuration creation"
|
||||
|
||||
# 等待DNS监控检测到配置文件
|
||||
echo "[INFO] 等待DNS监控检测并处理初始配置..."
|
||||
sleep 15
|
||||
|
||||
show_step "4" "Verify initial DNS configuration processing"
|
||||
# 检查两个容器的DNS监控日志
|
||||
check_dns_monitor_logs "logging-mvp-es-1"
|
||||
verify_step "Elasticsearch DNS monitor logs"
|
||||
|
||||
check_dns_monitor_logs "logging-mvp-kibana-1"
|
||||
verify_step "Kibana DNS monitor logs"
|
||||
|
||||
# 检查resolv.conf是否包含新的DNS服务器
|
||||
check_resolv_conf "logging-mvp-es-1" "8.8.8.8"
|
||||
verify_step "Elasticsearch resolv.conf initial check"
|
||||
|
||||
check_resolv_conf "logging-mvp-kibana-1" "8.8.8.8"
|
||||
verify_step "Kibana resolv.conf initial check"
|
||||
|
||||
show_step "5" "Modify DNS configuration and test auto-update"
|
||||
# 修改DNS配置文件 - 改为另一个IP
|
||||
echo "[INFO] 修改dns.conf文件，改为1.1.1.1..."
|
||||
cat > ./private/argus/etc/dns.conf << 'EOF'
|
||||
1.1.1.1
|
||||
EOF
|
||||
|
||||
echo "✅ dns.conf文件更新成功，改为1.1.1.1"
|
||||
|
||||
# 等待DNS监控检测到配置变化
|
||||
echo "[INFO] 等待DNS监控检测配置变化并执行更新..."
|
||||
sleep 15
|
||||
|
||||
show_step "6" "Verify DNS configuration auto-update"
|
||||
# 再次检查DNS监控日志，应该看到配置变化检测
|
||||
echo "[INFO] 检查DNS监控是否检测到配置变化..."
|
||||
|
||||
# 检查elasticsearch容器
|
||||
echo "[INFO] 检查elasticsearch容器的DNS监控日志（最近30行）..."
|
||||
docker exec logging-mvp-es-1 tail -n 30 /var/log/supervisor/dns-monitor.log || true
|
||||
|
||||
# 检查kibana容器
|
||||
echo "[INFO] 检查kibana容器的DNS监控日志（最近30行）..."
|
||||
docker exec logging-mvp-kibana-1 tail -n 30 /var/log/supervisor/dns-monitor.log || true
|
||||
|
||||
# 验证新的DNS服务器是否被添加到resolv.conf
|
||||
check_resolv_conf "logging-mvp-es-1" "1.1.1.1"
|
||||
verify_step "Elasticsearch resolv.conf after update"
|
||||
|
||||
check_resolv_conf "logging-mvp-kibana-1" "1.1.1.1"
|
||||
verify_step "Kibana resolv.conf after update"
|
||||
|
||||
show_step "7" "Final verification - Check DNS configuration"
|
||||
# 最终验证DNS配置
|
||||
echo "[INFO] 最终验证elasticsearch容器的resolv.conf..."
|
||||
docker exec logging-mvp-es-1 cat /etc/resolv.conf
|
||||
|
||||
echo "[INFO] 最终验证kibana容器的resolv.conf..."
|
||||
docker exec logging-mvp-kibana-1 cat /etc/resolv.conf
|
||||
|
||||
echo "[INFO] 最终dns.conf内容:"
|
||||
cat ./private/argus/etc/dns.conf
|
||||
|
||||
verify_step "Final DNS configuration verification"
|
||||
|
||||
show_step "8" "Cleanup - Stop all services"
|
||||
./scripts/05_down.sh
|
||||
verify_step "Service cleanup"
|
||||
|
||||
# 清理测试文件
|
||||
rm -f ./private/argus/etc/dns.conf
|
||||
# 注：不删除update-dns.sh，因为这是真实的脚本
|
||||
|
||||
# 计算总测试时间
|
||||
test_end_time=$(date +%s)
|
||||
total_time=$((test_end_time - test_start_time))
|
||||
|
||||
echo ""
|
||||
echo "======================================="
|
||||
echo "🎉 DNS监控功能测试完成!"
|
||||
echo "======================================="
|
||||
echo "📊 测试总结:"
|
||||
echo " • 总耗时: ${total_time}秒"
|
||||
echo " • 初始DNS配置: 8.8.8.8"
|
||||
echo " • 更新DNS配置: 1.1.1.1"
|
||||
echo " • DNS监控脚本正常工作"
|
||||
echo " • 容器resolv.conf自动覆盖更新成功"
|
||||
echo ""
|
||||
echo "✅ DNS自动更新功能测试通过!"
|
||||
echo ""
|
||||
188
src/log/tests/scripts/e2e_test.sh
Executable file
188
src/log/tests/scripts/e2e_test.sh
Executable file
@@ -0,0 +1,188 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "======================================="
|
||||
echo "ARGUS Log System End-to-End Test"
|
||||
echo "======================================="
|
||||
echo ""
|
||||
|
||||
# 记录测试开始时间
|
||||
test_start_time=$(date +%s)
|
||||
|
||||
# 函数：获取ES中的日志计数
|
||||
get_log_count() {
|
||||
local train_count=$(curl -s "http://localhost:9200/train-*/_count" 2>/dev/null | grep -o '"count":[0-9]*' | cut -d':' -f2 || echo "0")
|
||||
local infer_count=$(curl -s "http://localhost:9200/infer-*/_count" 2>/dev/null | grep -o '"count":[0-9]*' | cut -d':' -f2 || echo "0")
|
||||
echo "$((train_count + infer_count))"
|
||||
}
|
||||
|
||||
# 函数：等待服务就绪
|
||||
wait_for_services() {
|
||||
echo "[INFO] Waiting for all services to be ready..."
|
||||
local max_attempts=${SERVICE_WAIT_ATTEMPTS:-120}
|
||||
local attempt=1
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if curl -fs http://localhost:9200/_cluster/health >/dev/null 2>&1 && \
|
||||
curl -fs http://localhost:5601/api/status >/dev/null 2>&1 && \
|
||||
curl -fs http://localhost:2020/api/v2/metrics >/dev/null 2>&1 && \
|
||||
curl -fs http://localhost:2021/api/v2/metrics >/dev/null 2>&1; then
|
||||
echo "[OK] All services are ready!"
|
||||
return 0
|
||||
fi
|
||||
echo " Waiting for services... ($attempt/$max_attempts)"
|
||||
sleep 5
|
||||
((attempt++))
|
||||
done
|
||||
|
||||
echo "[ERROR] Services not ready after $max_attempts attempts"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 函数：显示测试步骤
|
||||
show_step() {
|
||||
echo ""
|
||||
echo "🔄 Step $1: $2"
|
||||
echo "----------------------------------------"
|
||||
}
|
||||
|
||||
# 函数：验证步骤结果
|
||||
verify_step() {
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ $1 - SUCCESS"
|
||||
else
|
||||
echo "❌ $1 - FAILED"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 开始端到端测试
|
||||
show_step "1" "Bootstrap - Initialize environment"
|
||||
./scripts/01_bootstrap.sh
|
||||
verify_step "Bootstrap"
|
||||
|
||||
show_step "2" "Startup - Start all services"
|
||||
./scripts/02_up.sh
|
||||
verify_step "Service startup"
|
||||
|
||||
# 等待服务完全就绪
|
||||
wait_for_services || exit 1
|
||||
|
||||
# 记录发送测试数据前的日志计数
|
||||
initial_count=$(get_log_count)
|
||||
echo "[INFO] Initial log count: $initial_count"
|
||||
|
||||
show_step "3a" "Send test data - Host01"
|
||||
./scripts/03_send_test_host01.sh
|
||||
verify_step "Test data sending (host01)"
|
||||
|
||||
show_step "3b" "Send test data - Host02"
|
||||
./scripts/03_send_test_host02.sh
|
||||
verify_step "Test data sending (host02)"
|
||||
|
||||
# 等待数据被处理
|
||||
echo "[INFO] Waiting for data to be processed..."
|
||||
sleep 10
|
||||
|
||||
show_step "4" "Verify data - Query Elasticsearch"
|
||||
./scripts/04_query_es.sh
|
||||
verify_step "Data verification"
|
||||
|
||||
# 记录发送测试数据后的日志计数
|
||||
final_count=$(get_log_count)
|
||||
echo "[INFO] Final log count: $final_count"
|
||||
|
||||
# 验证日志数量是否增加
|
||||
if [ "$final_count" -gt "$initial_count" ]; then
|
||||
added_logs=$((final_count - initial_count))
|
||||
echo "✅ Log count verification - SUCCESS: Added $added_logs logs (from $initial_count to $final_count)"
|
||||
else
|
||||
echo "❌ Log count verification - FAILED: Expected count to increase, but got $initial_count -> $final_count"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 验证预期的最小日志数量（每个主机应该发送一些日志）
|
||||
expected_min_logs=4 # 至少应该有几条日志
|
||||
if [ "$final_count" -ge "$expected_min_logs" ]; then
|
||||
echo "✅ Minimum log threshold - SUCCESS: $final_count logs (>= $expected_min_logs expected)"
|
||||
else
|
||||
echo "❌ Minimum log threshold - FAILED: Only $final_count logs (>= $expected_min_logs expected)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查服务健康状态
|
||||
show_step "Health" "Check service health"
|
||||
echo "[INFO] Checking service health..."
|
||||
|
||||
# 检查 Elasticsearch 健康状态
|
||||
health_check_ok=1
|
||||
es_health=$(curl -s "http://localhost:9200/_cluster/health" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
|
||||
if [ "$es_health" = "green" ] || [ "$es_health" = "yellow" ]; then
|
||||
echo "✅ Elasticsearch health: $es_health"
|
||||
else
|
||||
echo "❌ Elasticsearch health: $es_health"
|
||||
health_check_ok=0
|
||||
fi
|
||||
|
||||
# 检查 Kibana 状态
|
||||
if curl -fs "http://localhost:5601/api/status" >/dev/null 2>&1; then
|
||||
kb_status="available"
|
||||
echo "✅ Kibana status: $kb_status"
|
||||
|
||||
data_views_json=$(curl -fs "http://localhost:5601/api/data_views" -H 'kbn-xsrf: true' 2>/dev/null || true)
|
||||
if echo "$data_views_json" | grep -F '"title":"train-*"' >/dev/null 2>&1 && \
|
||||
echo "$data_views_json" | grep -F '"title":"infer-*"' >/dev/null 2>&1; then
|
||||
echo "✅ Kibana data views: train-* and infer-* present"
|
||||
else
|
||||
echo "❌ Kibana data views missing: train-* or infer-*"
|
||||
health_check_ok=0
|
||||
fi
|
||||
else
|
||||
kb_status="unavailable"
|
||||
echo "⚠️ Kibana status: $kb_status"
|
||||
health_check_ok=0
|
||||
fi
|
||||
|
||||
# 检查 Fluent-Bit 指标
|
||||
fb_host01_uptime=$(curl -s "http://localhost:2020/api/v2/metrics" | grep "fluentbit_uptime" | head -1 | grep -o "[0-9]\+$" || echo "0")
|
||||
fb_host02_uptime=$(curl -s "http://localhost:2021/api/v2/metrics" | grep "fluentbit_uptime" | head -1 | grep -o "[0-9]\+$" || echo "0")
|
||||
|
||||
if [ "$fb_host01_uptime" -gt 0 ] && [ "$fb_host02_uptime" -gt 0 ]; then
|
||||
echo "✅ Fluent-Bit services: host01 uptime=${fb_host01_uptime}s, host02 uptime=${fb_host02_uptime}s"
|
||||
else
|
||||
echo "⚠️ Fluent-Bit services: host01 uptime=${fb_host01_uptime}s, host02 uptime=${fb_host02_uptime}s"
|
||||
health_check_ok=0
|
||||
fi
|
||||
|
||||
if [ "$health_check_ok" -eq 1 ]; then
|
||||
true
|
||||
else
|
||||
false
|
||||
fi
|
||||
|
||||
verify_step "Service health check"
|
||||
|
||||
show_step "5" "Cleanup - Stop all services"
|
||||
./scripts/05_down.sh
|
||||
verify_step "Service cleanup"
|
||||
|
||||
# 计算总测试时间
|
||||
test_end_time=$(date +%s)
|
||||
total_time=$((test_end_time - test_start_time))
|
||||
|
||||
echo ""
|
||||
echo "======================================="
|
||||
echo "🎉 END-TO-END TEST COMPLETED SUCCESSFULLY!"
|
||||
echo "======================================="
|
||||
echo "📊 Test Summary:"
|
||||
echo " • Initial logs: $initial_count"
|
||||
echo " • Final logs: $final_count"
|
||||
echo " • Added logs: $added_logs"
|
||||
echo " • Total time: ${total_time}s"
|
||||
echo " • ES health: $es_health"
|
||||
echo " • Kibana status: $kb_status"
|
||||
echo " • DNS resolv: ✅ Passed (ES domain verified)"
|
||||
echo " • All services started and stopped successfully"
|
||||
echo ""
|
||||
echo "✅ The ARGUS log system is working correctly!"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user