[#1] 构建镜像；USER root; 使用supervisor守护进程；端到端本地测试
This commit is contained in:
root
78
src/log/tests/docker-compose.yml
Normal file
78
src/log/tests/docker-compose.yml
Normal file
@@ -0,0 +1,78 @@
|
||||
version: "3.8"
|
||||
services:
|
||||
es:
|
||||
build:
|
||||
context: ../elasticsearch/build
|
||||
dockerfile: Dockerfile
|
||||
image: argus-elasticsearch:latest
|
||||
environment:
|
||||
- discovery.type=single-node
|
||||
- xpack.security.enabled=false
|
||||
- ES_JAVA_OPTS=-Xms512m -Xmx512m
|
||||
volumes:
|
||||
- ./private/argus/log/elasticsearch:/private/argus/log/elasticsearch
|
||||
ports: ["9200:9200"]
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fs http://localhost:9200 >/dev/null || exit 1"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 30
|
||||
|
||||
kibana:
|
||||
build:
|
||||
context: ../kibana/build
|
||||
dockerfile: Dockerfile
|
||||
image: argus-kibana:latest
|
||||
environment:
|
||||
- ELASTICSEARCH_HOSTS=http://es:9200
|
||||
volumes:
|
||||
- ./private/argus/log/kibana:/private/argus/log/kibana
|
||||
ports: ["5601:5601"]
|
||||
depends_on:
|
||||
es:
|
||||
condition: service_healthy
|
||||
|
||||
fluent-bit-host01:
|
||||
image: ubuntu:22.04
|
||||
environment:
|
||||
- CLUSTER=local
|
||||
- RACK=dev
|
||||
- HOSTNAME=host01
|
||||
- ES_HOST=es
|
||||
- ES_PORT=9200
|
||||
volumes:
|
||||
- ../fluent-bit/start-fluent-bit.sh:/private/start-fluent-bit.sh:ro
|
||||
- ../fluent-bit/fluent-bit-bundle.tar.gz:/private/fluent-bit-bundle.tar.gz:ro
|
||||
ports: ["2020:2020"]
|
||||
depends_on:
|
||||
es:
|
||||
condition: service_healthy
|
||||
command: /private/start-fluent-bit.sh
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fs http://localhost:2020/api/v2/metrics >/dev/null || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
|
||||
fluent-bit-host02:
|
||||
image: ubuntu:22.04
|
||||
environment:
|
||||
- CLUSTER=local
|
||||
- RACK=dev
|
||||
- HOSTNAME=host02
|
||||
- ES_HOST=es
|
||||
- ES_PORT=9200
|
||||
volumes:
|
||||
- ../fluent-bit/start-fluent-bit.sh:/private/start-fluent-bit.sh:ro
|
||||
- ../fluent-bit/fluent-bit-bundle.tar.gz:/private/fluent-bit-bundle.tar.gz:ro
|
||||
ports: ["2021:2020"]
|
||||
depends_on:
|
||||
es:
|
||||
condition: service_healthy
|
||||
command: /private/start-fluent-bit.sh
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "curl -fs http://localhost:2020/api/v2/metrics >/dev/null || exit 1"]
|
||||
interval: 15s
|
||||
timeout: 10s
|
||||
retries: 30
|
||||
|
||||
27
src/log/tests/scripts/01_bootstrap.sh
Executable file
27
src/log/tests/scripts/01_bootstrap.sh
Executable file
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../" && pwd)"
|
||||
|
||||
# 创建新的private目录结构 (基于argus目录结构)
|
||||
echo "[INFO] Creating private directory structure for supervisor-based containers..."
|
||||
mkdir -p "$root/private/argus/log/elasticsearch"
|
||||
mkdir -p "$root/private/argus/log/kibana"
|
||||
|
||||
# 设置数据目录权限（ES 和 Kibana 容器都使用 UID 1000）
|
||||
echo "[INFO] Setting permissions for data directories..."
|
||||
sudo chown -R 1000:1000 "$root/private/argus/log/elasticsearch" 2>/dev/null || true
|
||||
sudo chown -R 1000:1000 "$root/private/argus/log/kibana" 2>/dev/null || true
|
||||
|
||||
echo "[INFO] Supervisor-based containers will manage their own scripts and configurations"
|
||||
|
||||
# 检查fluent-bit相关文件是否存在
|
||||
if [[ ! -f "$root/../fluent-bit/fluent-bit-bundle.tar.gz" ]]; then
|
||||
echo "[WARN] fluent-bit/fluent-bit-bundle.tar.gz 不存在，请确保已创建该文件"
|
||||
fi
|
||||
|
||||
if [[ ! -f "$root/../fluent-bit/start-fluent-bit.sh" ]]; then
|
||||
echo "[WARN] fluent-bit/start-fluent-bit.sh 不存在，请确保已创建该启动脚本"
|
||||
fi
|
||||
|
||||
echo "[OK] 初始化完成: private/argus/log/{elasticsearch,kibana}"
|
||||
echo "[INFO] Fluent-bit files should be in fluent-bit/ directory"
|
||||
10
src/log/tests/scripts/02_up.sh
Executable file
10
src/log/tests/scripts/02_up.sh
Executable file
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
compose_cmd="docker compose"
|
||||
if ! $compose_cmd version >/dev/null 2>&1; then
|
||||
if command -v docker-compose >/dev/null 2>&1; then compose_cmd="docker-compose"; else
|
||||
echo "需要 Docker Compose，请安装后重试" >&2; exit 1; fi
|
||||
fi
|
||||
$compose_cmd -p logging-mvp up -d --remove-orphans
|
||||
echo "[OK] 服务已启动：ES http://localhost:9200 Kibana http://localhost:5601 Fluent-Bit host01 http://localhost:2020 Fluent-Bit host02 http://localhost:2021"
|
||||
31
src/log/tests/scripts/03_send_test_host01.sh
Executable file
31
src/log/tests/scripts/03_send_test_host01.sh
Executable file
@@ -0,0 +1,31 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# 获取fluent-bit-host01容器名称
|
||||
container_name="logging-mvp-fluent-bit-host01-1"
|
||||
|
||||
# 检查容器是否存在并运行
|
||||
if ! docker ps | grep -q "$container_name"; then
|
||||
echo "[ERROR] Fluent Bit容器 $container_name 未运行"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 创建日志目录
|
||||
docker exec "$container_name" mkdir -p /logs/train /logs/infer
|
||||
|
||||
# 写入训练日志 (host01)
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host01] training step=1 loss=1.23 model=bert\n' \"\$(date '+%F %T')\" >> /logs/train/train-demo.log"
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host01] training step=2 loss=1.15 model=bert\n' \"\$(date '+%F %T')\" >> /logs/train/train-demo.log"
|
||||
|
||||
# 写入推理日志 (host01)
|
||||
docker exec "$container_name" sh -c "printf '%s ERROR [host01] inference failed on batch=1\n' \"\$(date '+%F %T')\" >> /logs/infer/infer-demo.log"
|
||||
docker exec "$container_name" sh -c "cat <<'STACK' >> /logs/infer/infer-demo.log
|
||||
Traceback (most recent call last):
|
||||
File \"inference.py\", line 15, in <module>
|
||||
raise RuntimeError(\"CUDA out of memory on host01\")
|
||||
RuntimeError: CUDA out of memory on host01
|
||||
STACK"
|
||||
|
||||
echo "[OK] 已通过docker exec写入测试日志到 host01 容器内："
|
||||
echo " - /logs/train/train-demo.log"
|
||||
echo " - /logs/infer/infer-demo.log"
|
||||
27
src/log/tests/scripts/03_send_test_host02.sh
Executable file
27
src/log/tests/scripts/03_send_test_host02.sh
Executable file
@@ -0,0 +1,27 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# 获取fluent-bit-host02容器名称
|
||||
container_name="logging-mvp-fluent-bit-host02-1"
|
||||
|
||||
# 检查容器是否存在并运行
|
||||
if ! docker ps | grep -q "$container_name"; then
|
||||
echo "[ERROR] Fluent Bit容器 $container_name 未运行"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 创建日志目录
|
||||
docker exec "$container_name" mkdir -p /logs/train /logs/infer
|
||||
|
||||
# 写入训练日志 (host02)
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host02] training step=1 loss=1.45 model=gpt\n' \"\$(date '+%F %T')\" >> /logs/train/train-demo.log"
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host02] training step=2 loss=1.38 model=gpt\n' \"\$(date '+%F %T')\" >> /logs/train/train-demo.log"
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host02] training step=3 loss=1.32 model=gpt\n' \"\$(date '+%F %T')\" >> /logs/train/train-demo.log"
|
||||
|
||||
# 写入推理日志 (host02)
|
||||
docker exec "$container_name" sh -c "printf '%s WARN [host02] inference slow on batch=5 latency=2.3s\n' \"\$(date '+%F %T')\" >> /logs/infer/infer-demo.log"
|
||||
docker exec "$container_name" sh -c "printf '%s INFO [host02] inference completed batch=6 latency=0.8s\n' \"\$(date '+%F %T')\" >> /logs/infer/infer-demo.log"
|
||||
|
||||
echo "[OK] 已通过docker exec写入测试日志到 host02 容器内："
|
||||
echo " - /logs/train/train-demo.log"
|
||||
echo " - /logs/infer/infer-demo.log"
|
||||
7
src/log/tests/scripts/04_query_es.sh
Executable file
7
src/log/tests/scripts/04_query_es.sh
Executable file
@@ -0,0 +1,7 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
ES="${ES:-http://localhost:9200}"
|
||||
echo "[i] 查询 ES 端点：$ES"
|
||||
curl -fsS "$ES/_cat/indices?v" | egrep 'train-|infer-|logstash' || true
|
||||
printf "train-* 计数："; curl -fsS "$ES/train-*/_count" | sed -E 's/.*"count":([0-9]+).*/\1/'; echo
|
||||
printf "infer-* 计数："; curl -fsS "$ES/infer-*/_count" | sed -E 's/.*"count":([0-9]+).*/\1/'; echo
|
||||
10
src/log/tests/scripts/05_down.sh
Executable file
10
src/log/tests/scripts/05_down.sh
Executable file
@@ -0,0 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
compose_cmd="docker compose"
|
||||
if ! $compose_cmd version >/dev/null 2>&1; then
|
||||
if command -v docker-compose >/dev/null 2>&1; then compose_cmd="docker-compose"; else
|
||||
echo "需要 Docker Compose，请安装后重试" >&2; exit 1; fi
|
||||
fi
|
||||
$compose_cmd -p logging-mvp down
|
||||
echo "[OK] 已停止所有容器"
|
||||
168
src/log/tests/scripts/e2e_test.sh
Executable file
168
src/log/tests/scripts/e2e_test.sh
Executable file
@@ -0,0 +1,168 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "======================================="
|
||||
echo "ARGUS Log System End-to-End Test"
|
||||
echo "======================================="
|
||||
echo ""
|
||||
|
||||
# 记录测试开始时间
|
||||
test_start_time=$(date +%s)
|
||||
|
||||
# 函数：获取ES中的日志计数
|
||||
get_log_count() {
|
||||
local train_count=$(curl -s "http://localhost:9200/train-*/_count" 2>/dev/null | grep -o '"count":[0-9]*' | cut -d':' -f2 || echo "0")
|
||||
local infer_count=$(curl -s "http://localhost:9200/infer-*/_count" 2>/dev/null | grep -o '"count":[0-9]*' | cut -d':' -f2 || echo "0")
|
||||
echo "$((train_count + infer_count))"
|
||||
}
|
||||
|
||||
# 函数：等待服务就绪
|
||||
wait_for_services() {
|
||||
echo "[INFO] Waiting for all services to be ready..."
|
||||
local max_attempts=60
|
||||
local attempt=1
|
||||
|
||||
while [ $attempt -le $max_attempts ]; do
|
||||
if curl -fs http://localhost:9200/_cluster/health >/dev/null 2>&1 && \
|
||||
curl -fs http://localhost:5601/api/status >/dev/null 2>&1 && \
|
||||
curl -fs http://localhost:2020/api/v2/metrics >/dev/null 2>&1 && \
|
||||
curl -fs http://localhost:2021/api/v2/metrics >/dev/null 2>&1; then
|
||||
echo "[OK] All services are ready!"
|
||||
return 0
|
||||
fi
|
||||
echo " Waiting for services... ($attempt/$max_attempts)"
|
||||
sleep 5
|
||||
((attempt++))
|
||||
done
|
||||
|
||||
echo "[ERROR] Services not ready after $max_attempts attempts"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 函数：显示测试步骤
|
||||
show_step() {
|
||||
echo ""
|
||||
echo "🔄 Step $1: $2"
|
||||
echo "----------------------------------------"
|
||||
}
|
||||
|
||||
# 函数：验证步骤结果
|
||||
verify_step() {
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "✅ $1 - SUCCESS"
|
||||
else
|
||||
echo "❌ $1 - FAILED"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# 开始端到端测试
|
||||
show_step "1" "Bootstrap - Initialize environment"
|
||||
./scripts/01_bootstrap.sh
|
||||
verify_step "Bootstrap"
|
||||
|
||||
show_step "2" "Startup - Start all services"
|
||||
./scripts/02_up.sh
|
||||
verify_step "Service startup"
|
||||
|
||||
# 等待服务完全就绪
|
||||
wait_for_services || exit 1
|
||||
|
||||
# 记录发送测试数据前的日志计数
|
||||
initial_count=$(get_log_count)
|
||||
echo "[INFO] Initial log count: $initial_count"
|
||||
|
||||
show_step "3a" "Send test data - Host01"
|
||||
./scripts/03_send_test_host01.sh
|
||||
verify_step "Test data sending (host01)"
|
||||
|
||||
show_step "3b" "Send test data - Host02"
|
||||
./scripts/03_send_test_host02.sh
|
||||
verify_step "Test data sending (host02)"
|
||||
|
||||
# 等待数据被处理
|
||||
echo "[INFO] Waiting for data to be processed..."
|
||||
sleep 10
|
||||
|
||||
show_step "4" "Verify data - Query Elasticsearch"
|
||||
./scripts/04_query_es.sh
|
||||
verify_step "Data verification"
|
||||
|
||||
# 记录发送测试数据后的日志计数
|
||||
final_count=$(get_log_count)
|
||||
echo "[INFO] Final log count: $final_count"
|
||||
|
||||
# 验证日志数量是否增加
|
||||
if [ "$final_count" -gt "$initial_count" ]; then
|
||||
added_logs=$((final_count - initial_count))
|
||||
echo "✅ Log count verification - SUCCESS: Added $added_logs logs (from $initial_count to $final_count)"
|
||||
else
|
||||
echo "❌ Log count verification - FAILED: Expected count to increase, but got $initial_count -> $final_count"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 验证预期的最小日志数量（每个主机应该发送一些日志）
|
||||
expected_min_logs=4 # 至少应该有几条日志
|
||||
if [ "$final_count" -ge "$expected_min_logs" ]; then
|
||||
echo "✅ Minimum log threshold - SUCCESS: $final_count logs (>= $expected_min_logs expected)"
|
||||
else
|
||||
echo "❌ Minimum log threshold - FAILED: Only $final_count logs (>= $expected_min_logs expected)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 检查服务健康状态
|
||||
show_step "Health" "Check service health"
|
||||
echo "[INFO] Checking service health..."
|
||||
|
||||
# 检查 Elasticsearch 健康状态
|
||||
es_health=$(curl -s "http://localhost:9200/_cluster/health" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
|
||||
if [ "$es_health" = "green" ] || [ "$es_health" = "yellow" ]; then
|
||||
echo "✅ Elasticsearch health: $es_health"
|
||||
else
|
||||
echo "❌ Elasticsearch health: $es_health"
|
||||
fi
|
||||
|
||||
# 检查 Kibana 状态
|
||||
if curl -fs "http://localhost:5601/api/status" >/dev/null 2>&1; then
|
||||
kb_status="available"
|
||||
echo "✅ Kibana status: $kb_status"
|
||||
else
|
||||
kb_status="unavailable"
|
||||
echo "⚠️ Kibana status: $kb_status"
|
||||
fi
|
||||
|
||||
# 检查 Fluent-Bit 指标
|
||||
fb_host01_uptime=$(curl -s "http://localhost:2020/api/v2/metrics" | grep "fluentbit_uptime" | head -1 | grep -o "[0-9]\+$" || echo "0")
|
||||
fb_host02_uptime=$(curl -s "http://localhost:2021/api/v2/metrics" | grep "fluentbit_uptime" | head -1 | grep -o "[0-9]\+$" || echo "0")
|
||||
|
||||
if [ "$fb_host01_uptime" -gt 0 ] && [ "$fb_host02_uptime" -gt 0 ]; then
|
||||
echo "✅ Fluent-Bit services: host01 uptime=${fb_host01_uptime}s, host02 uptime=${fb_host02_uptime}s"
|
||||
else
|
||||
echo "⚠️ Fluent-Bit services: host01 uptime=${fb_host01_uptime}s, host02 uptime=${fb_host02_uptime}s"
|
||||
fi
|
||||
|
||||
verify_step "Service health check"
|
||||
|
||||
show_step "5" "Cleanup - Stop all services"
|
||||
./scripts/05_down.sh
|
||||
verify_step "Service cleanup"
|
||||
|
||||
# 计算总测试时间
|
||||
test_end_time=$(date +%s)
|
||||
total_time=$((test_end_time - test_start_time))
|
||||
|
||||
echo ""
|
||||
echo "======================================="
|
||||
echo "🎉 END-TO-END TEST COMPLETED SUCCESSFULLY!"
|
||||
echo "======================================="
|
||||
echo "📊 Test Summary:"
|
||||
echo " • Initial logs: $initial_count"
|
||||
echo " • Final logs: $final_count"
|
||||
echo " • Added logs: $added_logs"
|
||||
echo " • Total time: ${total_time}s"
|
||||
echo " • ES health: $es_health"
|
||||
echo " • Kibana status: $kb_status"
|
||||
echo " • All services started and stopped successfully"
|
||||
echo ""
|
||||
echo "✅ The ARGUS log system is working correctly!"
|
||||
echo ""
|
||||
Reference in New Issue
Block a user