Compare commits
2 Commits
6771398b5b
...
3e85e50e78
Author | SHA1 | Date | |
---|---|---|---|
|
3e85e50e78 | ||
|
f8814cd705 |
@ -27,9 +27,10 @@ COPY db.argus.com /etc/bind/db.argus.com
|
||||
# Copy startup and reload scripts
|
||||
COPY startup.sh /usr/local/bin/startup.sh
|
||||
COPY reload-bind9.sh /usr/local/bin/reload-bind9.sh
|
||||
COPY argus_dns_sync.sh /usr/local/bin/argus_dns_sync.sh
|
||||
|
||||
# Make scripts executable
|
||||
RUN chmod +x /usr/local/bin/startup.sh /usr/local/bin/reload-bind9.sh
|
||||
RUN chmod +x /usr/local/bin/startup.sh /usr/local/bin/reload-bind9.sh /usr/local/bin/argus_dns_sync.sh
|
||||
|
||||
# Set proper ownership for BIND9 files
|
||||
RUN chown bind:bind /etc/bind/named.conf.local /etc/bind/db.argus.com
|
||||
@ -41,4 +42,4 @@ EXPOSE 53/tcp 53/udp
|
||||
USER root
|
||||
|
||||
# Start with startup script
|
||||
CMD ["/usr/local/bin/startup.sh"]
|
||||
CMD ["/usr/local/bin/startup.sh"]
|
||||
|
100
src/bind/build/argus_dns_sync.sh
Normal file
100
src/bind/build/argus_dns_sync.sh
Normal file
@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
WATCH_DIR="/private/argus/etc"
|
||||
ZONE_DB="/private/argus/bind/db.argus.com"
|
||||
LOCKFILE="/var/lock/argus_dns_sync.lock"
|
||||
BACKUP_DIR="/private/argus/bind/.backup"
|
||||
SLEEP_SECONDS=10
|
||||
RELOAD_SCRIPT="/usr/local/bin/reload-bind9.sh" # 这里放你已有脚本的路径
|
||||
|
||||
mkdir -p "$(dirname "$LOCKFILE")" "$BACKUP_DIR"
|
||||
|
||||
is_ipv4() {
|
||||
local ip="$1"
|
||||
[[ "$ip" =~ ^([0-9]{1,3}\.){3}[0-9]{1,3}$ ]] || return 1
|
||||
IFS='.' read -r a b c d <<<"$ip"
|
||||
for n in "$a" "$b" "$c" "$d"; do
|
||||
(( n >= 0 && n <= 255 )) || return 1
|
||||
done
|
||||
return 0
|
||||
}
|
||||
|
||||
get_current_ip() {
|
||||
local name="$1"
|
||||
sed -n -E "s/^${name}[[:space:]]+IN[[:space:]]+A[[:space:]]+([0-9.]+)[[:space:]]*$/\1/p" "$ZONE_DB" | head -n1
|
||||
}
|
||||
|
||||
upsert_record() {
|
||||
local name="$1"
|
||||
local new_ip="$2"
|
||||
local ts
|
||||
ts="$(date +%Y%m%d-%H%M%S)"
|
||||
local changed=0
|
||||
|
||||
cp -a "$ZONE_DB" "$BACKUP_DIR/db.argus.com.$ts.bak"
|
||||
|
||||
local cur_ip
|
||||
cur_ip="$(get_current_ip "$name" || true)"
|
||||
|
||||
if [[ -z "$cur_ip" ]]; then
|
||||
# Ensure the file ends with a newline before adding new record
|
||||
if [[ -s "$ZONE_DB" ]] && [[ $(tail -c1 "$ZONE_DB" | wc -l) -eq 0 ]]; then
|
||||
echo "" >> "$ZONE_DB"
|
||||
fi
|
||||
printf "%-20s IN A %s\n" "$name" "$new_ip" >> "$ZONE_DB"
|
||||
echo "[ADD] ${name} -> ${new_ip}"
|
||||
changed=1
|
||||
elif [[ "$cur_ip" != "$new_ip" ]]; then
|
||||
awk -v n="$name" -v ip="$new_ip" '
|
||||
{
|
||||
if ($1==n && $2=="IN" && $3=="A") {
|
||||
printf "%-20s IN A %s\n", n, ip
|
||||
} else {
|
||||
print
|
||||
}
|
||||
}
|
||||
' "$ZONE_DB" > "${ZONE_DB}.tmp" && mv "${ZONE_DB}.tmp" "$ZONE_DB"
|
||||
echo "[UPDATE] ${name}: ${cur_ip} -> ${new_ip}"
|
||||
changed=1
|
||||
else
|
||||
echo "[SKIP] ${name} unchanged (${new_ip})"
|
||||
fi
|
||||
|
||||
return $changed
|
||||
}
|
||||
|
||||
while true; do
|
||||
exec 9>"$LOCKFILE"
|
||||
if flock -n 9; then
|
||||
shopt -s nullglob
|
||||
NEED_RELOAD=0
|
||||
|
||||
for f in "$WATCH_DIR"/*.argus.com; do
|
||||
base="$(basename "$f")"
|
||||
name="${base%.argus.com}"
|
||||
ip="$(grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' "$f" | tail -n1 || true)"
|
||||
|
||||
if [[ -z "$ip" ]] || ! is_ipv4 "$ip"; then
|
||||
echo "[WARN] $f 未找到有效 IPv4,跳过"
|
||||
continue
|
||||
fi
|
||||
|
||||
if upsert_record "$name" "$ip"; then
|
||||
NEED_RELOAD=1
|
||||
fi
|
||||
done
|
||||
|
||||
if [[ $NEED_RELOAD -eq 1 ]]; then
|
||||
echo "[INFO] 检测到 db.argus.com 变更,执行 reload-bind9.sh"
|
||||
bash "$RELOAD_SCRIPT"
|
||||
fi
|
||||
|
||||
flock -u 9
|
||||
else
|
||||
echo "[INFO] 已有同步任务在运行,跳过本轮"
|
||||
fi
|
||||
|
||||
sleep "$SLEEP_SECONDS"
|
||||
done
|
||||
|
@ -3,8 +3,9 @@
|
||||
# Set /private permissions to 777 as requested
|
||||
chmod 777 /private 2>/dev/null || true
|
||||
|
||||
# Create persistent directory for BIND9 configs
|
||||
# Create persistent directories for BIND9 configs and DNS sync
|
||||
mkdir -p /private/argus/bind
|
||||
mkdir -p /private/argus/etc
|
||||
|
||||
# Copy configuration files to persistent storage if they don't exist
|
||||
if [ ! -f /private/argus/bind/named.conf.local ]; then
|
||||
|
@ -21,4 +21,17 @@ autostart=true
|
||||
autorestart=true
|
||||
stderr_logfile=/var/log/supervisor/bind9.err.log
|
||||
stdout_logfile=/var/log/supervisor/bind9.out.log
|
||||
priority=10
|
||||
priority=10
|
||||
|
||||
[program:argus-dns-sync]
|
||||
command=/usr/local/bin/argus_dns_sync.sh
|
||||
autostart=true
|
||||
autorestart=true
|
||||
startsecs=3
|
||||
stopsignal=TERM
|
||||
user=root
|
||||
stdout_logfile=/var/log/argus_dns_sync.out.log
|
||||
stderr_logfile=/var/log/argus_dns_sync.err.log
|
||||
; 根据环境调整环境变量(可选)
|
||||
; environment=RNDC_RELOAD="yes"
|
||||
|
||||
|
@ -1,16 +0,0 @@
|
||||
$TTL 604800
|
||||
@ IN SOA ns1.argus.com. admin.argus.com. (
|
||||
3 ; Serial
|
||||
604800 ; Refresh
|
||||
86400 ; Retry
|
||||
2419200 ; Expire
|
||||
604800 ) ; Negative Cache TTL
|
||||
|
||||
; 定义 DNS 服务器
|
||||
@ IN NS ns1.argus.com.
|
||||
|
||||
; 定义 ns1 主机
|
||||
ns1 IN A 127.0.0.1
|
||||
|
||||
; 定义 web 指向 192.168.1.100
|
||||
web IN A 192.168.1.100
|
@ -1,16 +0,0 @@
|
||||
$TTL 604800
|
||||
@ IN SOA ns1.argus.com. admin.argus.com. (
|
||||
2 ; Serial
|
||||
604800 ; Refresh
|
||||
86400 ; Retry
|
||||
2419200 ; Expire
|
||||
604800 ) ; Negative Cache TTL
|
||||
|
||||
; 定义 DNS 服务器
|
||||
@ IN NS ns1.argus.com.
|
||||
|
||||
; 定义 ns1 主机
|
||||
ns1 IN A 127.0.0.1
|
||||
|
||||
; 定义 web 指向 12.4.5.6
|
||||
web IN A 12.4.5.6
|
@ -1,4 +0,0 @@
|
||||
zone "argus.com" {
|
||||
type master;
|
||||
file "/etc/bind/db.argus.com";
|
||||
};
|
@ -69,6 +69,8 @@ run_test_step "TEST-02" "02_dig_test.sh" "Initial DNS resolution test" || true
|
||||
|
||||
run_test_step "TEST-03" "03_reload_test.sh" "Configuration reload with IP modification" || true
|
||||
|
||||
run_test_step "TEST-03.5" "03.5_dns_sync_test.sh" "DNS auto-sync functionality test" || true
|
||||
|
||||
run_test_step "TEST-04" "04_persistence_test.sh" "Configuration persistence after restart" || true
|
||||
|
||||
# Final cleanup (but preserve logs for review)
|
||||
@ -93,6 +95,7 @@ if [ $failed_tests -eq 0 ]; then
|
||||
echo " ✓ Container startup and basic functionality"
|
||||
echo " ✓ DNS resolution for configured domains"
|
||||
echo " ✓ Configuration modification and reload"
|
||||
echo " ✓ DNS auto-sync from IP files"
|
||||
echo " ✓ Configuration persistence across restarts"
|
||||
echo " ✓ Cleanup and resource management"
|
||||
echo ""
|
||||
|
256
src/bind/tests/scripts/03.5_dns_sync_test.sh
Executable file
256
src/bind/tests/scripts/03.5_dns_sync_test.sh
Executable file
@ -0,0 +1,256 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test DNS auto-sync functionality using argus_dns_sync.sh
|
||||
# This test validates the automatic DNS record updates from IP files
|
||||
# Usage: ./03.5_dns_sync_test.sh
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TEST_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
echo "=== DNS Auto-Sync Functionality Test ==="
|
||||
|
||||
# Check if container is running
|
||||
if ! docker compose ps | grep -q "Up"; then
|
||||
echo "Error: BIND9 container is not running"
|
||||
echo "Please start the container first with: ./01_start_container.sh"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if dig is available
|
||||
if ! command -v dig &> /dev/null; then
|
||||
echo "Installing dig (dnsutils)..."
|
||||
apt-get update && apt-get install -y dnsutils
|
||||
fi
|
||||
|
||||
# Function to test DNS query
|
||||
test_dns_query() {
|
||||
local hostname="$1"
|
||||
local expected_ip="$2"
|
||||
local description="$3"
|
||||
|
||||
echo "Testing: $description"
|
||||
echo "Query: $hostname.argus.com -> Expected: $expected_ip"
|
||||
|
||||
# Wait a moment for DNS cache
|
||||
sleep 2
|
||||
|
||||
result=$(dig @localhost $hostname.argus.com A +short 2>/dev/null || echo "QUERY_FAILED")
|
||||
|
||||
if [ "$result" = "$expected_ip" ]; then
|
||||
echo "✓ $result"
|
||||
return 0
|
||||
else
|
||||
echo "✗ Got: $result, Expected: $expected_ip"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to wait for sync to complete
|
||||
wait_for_sync() {
|
||||
local timeout=15
|
||||
local elapsed=0
|
||||
echo "Waiting for DNS sync to complete (max ${timeout}s)..."
|
||||
|
||||
while [ $elapsed -lt $timeout ]; do
|
||||
if docker compose exec bind9 test -f /var/lock/argus_dns_sync.lock; then
|
||||
echo "Sync process is running..."
|
||||
else
|
||||
echo "Sync completed"
|
||||
sleep 2 # Extra wait for DNS propagation
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
elapsed=$((elapsed + 2))
|
||||
done
|
||||
|
||||
echo "Warning: Sync may still be running after ${timeout}s"
|
||||
return 0
|
||||
}
|
||||
|
||||
echo ""
|
||||
echo "Step 1: Preparing test environment..."
|
||||
|
||||
# Ensure required directories exist
|
||||
docker compose exec bind9 mkdir -p /private/argus/etc
|
||||
docker compose exec bind9 mkdir -p /private/argus/bind/.backup
|
||||
|
||||
# Backup original configuration if it exists
|
||||
docker compose exec bind9 test -f /private/argus/bind/db.argus.com && \
|
||||
docker compose exec bind9 cp /private/argus/bind/db.argus.com /private/argus/bind/db.argus.com.backup.test || true
|
||||
|
||||
# Ensure initial configuration is available (may already be symlinked)
|
||||
docker compose exec bind9 test -f /private/argus/bind/db.argus.com || \
|
||||
docker compose exec bind9 cp /etc/bind/db.argus.com /private/argus/bind/db.argus.com
|
||||
|
||||
echo "✓ Test environment prepared"
|
||||
|
||||
echo ""
|
||||
echo "Step 2: Testing initial DNS configuration..."
|
||||
|
||||
# Get current IP for web.argus.com (may have been changed by previous tests)
|
||||
current_web_ip=$(dig @localhost web.argus.com A +short 2>/dev/null || echo "UNKNOWN")
|
||||
echo "Current web.argus.com IP: $current_web_ip"
|
||||
|
||||
# Test that DNS is working (regardless of specific IP)
|
||||
if [ "$current_web_ip" = "UNKNOWN" ] || [ -z "$current_web_ip" ]; then
|
||||
echo "DNS resolution not working for web.argus.com"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "✓ DNS resolution is working"
|
||||
|
||||
echo ""
|
||||
echo "Step 3: Creating IP files for auto-sync..."
|
||||
|
||||
# Create test IP files in the watch directory
|
||||
echo "Creating test1.argus.com with IP 10.0.0.100"
|
||||
docker compose exec bind9 bash -c 'echo "10.0.0.100" > /private/argus/etc/test1.argus.com'
|
||||
|
||||
echo "Creating test2.argus.com with IP 10.0.0.200"
|
||||
docker compose exec bind9 bash -c 'echo "test2 service running on 10.0.0.200" > /private/argus/etc/test2.argus.com'
|
||||
|
||||
echo "Creating api.argus.com with IP 192.168.1.50"
|
||||
docker compose exec bind9 bash -c 'echo "API server: 192.168.1.50 port 8080" > /private/argus/etc/api.argus.com'
|
||||
|
||||
echo "✓ IP files created"
|
||||
|
||||
echo ""
|
||||
echo "Step 4: Checking DNS sync process..."
|
||||
|
||||
# Check if DNS sync process is already running (via supervisord)
|
||||
if docker compose exec bind9 pgrep -f argus_dns_sync.sh > /dev/null; then
|
||||
echo "✓ DNS sync process already running (via supervisord)"
|
||||
else
|
||||
echo "Starting DNS sync process manually..."
|
||||
# Start the DNS sync process in background if not running
|
||||
docker compose exec -d bind9 /usr/local/bin/argus_dns_sync.sh
|
||||
echo "✓ DNS sync process started manually"
|
||||
fi
|
||||
|
||||
# Wait for first sync cycle
|
||||
wait_for_sync
|
||||
|
||||
echo ""
|
||||
echo "Step 5: Testing auto-synced DNS records..."
|
||||
|
||||
failed_tests=0
|
||||
|
||||
# Test new DNS records created by auto-sync
|
||||
if ! test_dns_query "test1" "10.0.0.100" "Auto-synced test1.argus.com"; then
|
||||
((failed_tests++))
|
||||
fi
|
||||
|
||||
if ! test_dns_query "test2" "10.0.0.200" "Auto-synced test2.argus.com"; then
|
||||
((failed_tests++))
|
||||
fi
|
||||
|
||||
if ! test_dns_query "api" "192.168.1.50" "Auto-synced api.argus.com"; then
|
||||
((failed_tests++))
|
||||
fi
|
||||
|
||||
# Verify original records still work (use current IP from earlier)
|
||||
if ! test_dns_query "web" "$current_web_ip" "Original web.argus.com still working"; then
|
||||
((failed_tests++))
|
||||
fi
|
||||
|
||||
if ! test_dns_query "ns1" "127.0.0.1" "Original ns1.argus.com still working"; then
|
||||
((failed_tests++))
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 6: Testing IP update functionality..."
|
||||
|
||||
# Update an existing IP file
|
||||
echo "Updating test1.argus.com IP from 10.0.0.100 to 10.0.0.150"
|
||||
docker compose exec bind9 bash -c 'echo "10.0.0.150" > /private/argus/etc/test1.argus.com'
|
||||
|
||||
# Wait for sync
|
||||
wait_for_sync
|
||||
|
||||
# Test updated record
|
||||
if ! test_dns_query "test1" "10.0.0.150" "Updated test1.argus.com IP"; then
|
||||
((failed_tests++))
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 7: Testing invalid IP handling..."
|
||||
|
||||
# Create file with invalid IP
|
||||
echo "Creating invalid.argus.com with invalid IP"
|
||||
docker compose exec bind9 bash -c 'echo "this is not an IP address" > /private/argus/etc/invalid.argus.com'
|
||||
|
||||
# Wait for sync (should skip invalid IP)
|
||||
wait_for_sync
|
||||
|
||||
# Verify invalid record was not added (should fail to resolve)
|
||||
result=$(dig @localhost invalid.argus.com A +short 2>/dev/null || echo "NO_RESULT")
|
||||
if [ "$result" = "NO_RESULT" ] || [ -z "$result" ]; then
|
||||
echo "✓ Invalid IP correctly ignored"
|
||||
else
|
||||
echo "✗ Invalid IP was processed: $result"
|
||||
((failed_tests++))
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 8: Verifying backup functionality..."
|
||||
|
||||
# Check if backups were created
|
||||
backup_count=$(docker compose exec bind9 ls -1 /private/argus/bind/.backup/ | wc -l || echo "0")
|
||||
if [ "$backup_count" -gt 0 ]; then
|
||||
echo "✓ Configuration backups created ($backup_count files)"
|
||||
# Show latest backup
|
||||
docker compose exec bind9 ls -la /private/argus/bind/.backup/ | tail -1
|
||||
else
|
||||
echo "✗ No backup files found"
|
||||
((failed_tests++))
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 9: Cleanup..."
|
||||
|
||||
# Note: We don't stop the DNS sync process since it's managed by supervisord
|
||||
echo "Note: DNS sync process will continue running (managed by supervisord)"
|
||||
|
||||
# Clean up test files
|
||||
docker compose exec bind9 rm -f /private/argus/etc/test1.argus.com
|
||||
docker compose exec bind9 rm -f /private/argus/etc/test2.argus.com
|
||||
docker compose exec bind9 rm -f /private/argus/etc/api.argus.com
|
||||
docker compose exec bind9 rm -f /private/argus/etc/invalid.argus.com
|
||||
|
||||
# Restore original configuration if backup exists
|
||||
docker compose exec bind9 test -f /private/argus/bind/db.argus.com.backup.test && \
|
||||
docker compose exec bind9 cp /private/argus/bind/db.argus.com.backup.test /private/argus/bind/db.argus.com && \
|
||||
docker compose exec bind9 rm /private/argus/bind/db.argus.com.backup.test || true
|
||||
|
||||
# Reload original configuration
|
||||
docker compose exec bind9 /usr/local/bin/reload-bind9.sh
|
||||
|
||||
echo "✓ Cleanup completed"
|
||||
|
||||
echo ""
|
||||
echo "=== DNS Auto-Sync Test Summary ==="
|
||||
if [ $failed_tests -eq 0 ]; then
|
||||
echo "✅ All DNS auto-sync tests passed!"
|
||||
echo ""
|
||||
echo "Validated functionality:"
|
||||
echo " ✓ Automatic DNS record creation from IP files"
|
||||
echo " ✓ IP address extraction from various file formats"
|
||||
echo " ✓ Dynamic DNS record updates"
|
||||
echo " ✓ Invalid IP address handling"
|
||||
echo " ✓ Configuration backup mechanism"
|
||||
echo " ✓ Preservation of existing DNS records"
|
||||
echo ""
|
||||
echo "The DNS auto-sync functionality is working correctly!"
|
||||
exit 0
|
||||
else
|
||||
echo "❌ $failed_tests DNS auto-sync test(s) failed!"
|
||||
echo ""
|
||||
echo "Please check:"
|
||||
echo " - argus_dns_sync.sh script configuration"
|
||||
echo " - File permissions in /private/argus/etc/"
|
||||
echo " - BIND9 reload functionality"
|
||||
echo " - Network connectivity and DNS resolution"
|
||||
exit 1
|
||||
fi
|
@ -9,7 +9,7 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TEST_DIR="$(dirname "$SCRIPT_DIR")"
|
||||
|
||||
# Parse command line arguments
|
||||
FULL_CLEANUP=false
|
||||
FULL_CLEANUP=true
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case $1 in
|
||||
--full)
|
||||
@ -19,7 +19,7 @@ while [[ $# -gt 0 ]]; do
|
||||
*)
|
||||
echo "Unknown option: $1"
|
||||
echo "Usage: $0 [--full]"
|
||||
echo " --full: Also remove persistent data and Docker image"
|
||||
echo " --full: Also remove persistent data "
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
@ -57,24 +57,6 @@ if [ "$FULL_CLEANUP" = true ]; then
|
||||
echo "✓ No persistent data directory found"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 4: Removing Docker image..."
|
||||
|
||||
# Remove the test image if it exists
|
||||
if docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "^argus-bind9:latest$"; then
|
||||
docker rmi argus-bind9:latest
|
||||
echo "✓ Docker image 'argus-bind9:latest' removed"
|
||||
else
|
||||
echo "✓ Docker image 'argus-bind9:latest' not found"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Step 5: Cleaning up unused Docker resources..."
|
||||
|
||||
# Clean up any dangling images and unused volumes
|
||||
docker system prune -f > /dev/null 2>&1 || true
|
||||
|
||||
echo "✓ Docker system cleaned"
|
||||
else
|
||||
echo ""
|
||||
echo "Step 3: Preserving persistent data and Docker image..."
|
||||
@ -92,8 +74,6 @@ echo "✓ Docker networks cleaned"
|
||||
|
||||
if [ "$FULL_CLEANUP" = true ]; then
|
||||
echo "✓ Persistent data removed"
|
||||
echo "✓ Docker image removed"
|
||||
echo "✓ System resources cleaned"
|
||||
echo ""
|
||||
echo "Full cleanup completed! Test environment completely removed."
|
||||
else
|
||||
@ -104,4 +84,4 @@ else
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "Test environment cleanup finished."
|
||||
echo "Test environment cleanup finished."
|
||||
|
37
src/log/fluent-bit/build/etc/fluent-bit.conf
Normal file
37
src/log/fluent-bit/build/etc/fluent-bit.conf
Normal file
@ -0,0 +1,37 @@
|
||||
[SERVICE]
|
||||
Daemon Off
|
||||
Parsers_File parsers.conf
|
||||
HTTP_Server On
|
||||
HTTP_Listen 0.0.0.0
|
||||
HTTP_Port 2020
|
||||
storage.path /buffers
|
||||
storage.sync normal
|
||||
storage.checksum on
|
||||
storage.backlog.mem_limit 128M
|
||||
# 备注:该镜像默认未开启 Hot Reload,修改配置后请重启容器。
|
||||
|
||||
@INCLUDE inputs.d/*.conf
|
||||
|
||||
[FILTER]
|
||||
Name parser
|
||||
Match app.*
|
||||
Key_Name log
|
||||
Parser timestamp_parser
|
||||
Reserve_Data On
|
||||
Preserve_Key On
|
||||
Unescape_Key On
|
||||
|
||||
[FILTER]
|
||||
Name record_modifier
|
||||
Match *
|
||||
Record cluster ${CLUSTER}
|
||||
Record rack ${RACK}
|
||||
Record host ${HOSTNAME}
|
||||
|
||||
[FILTER]
|
||||
Name lua
|
||||
Match app.*
|
||||
script inject_labels.lua
|
||||
call add_labels
|
||||
|
||||
@INCLUDE outputs.d/*.conf
|
15
src/log/fluent-bit/build/etc/inject_labels.lua
Normal file
15
src/log/fluent-bit/build/etc/inject_labels.lua
Normal file
@ -0,0 +1,15 @@
|
||||
function add_labels(tag, ts, record)
|
||||
record["job_id"] = os.getenv("FB_JOB_ID") or record["job_id"] or "unknown"
|
||||
record["user"] = os.getenv("FB_USER") or record["user"] or "unknown"
|
||||
record["model"] = os.getenv("FB_MODEL") or record["model"] or "unknown"
|
||||
record["gpu_id"] = os.getenv("FB_GPU_ID") or record["gpu_id"] or "na"
|
||||
local p = record["log_path"] or ""
|
||||
if string.find(p, "/logs/infer/") then
|
||||
record["role"] = "infer"
|
||||
elseif string.find(p, "/logs/train/") then
|
||||
record["role"] = "train"
|
||||
else
|
||||
record["role"] = record["role"] or "app"
|
||||
end
|
||||
return 1, ts, record
|
||||
end
|
10
src/log/fluent-bit/build/etc/inputs.d/10-train.conf
Normal file
10
src/log/fluent-bit/build/etc/inputs.d/10-train.conf
Normal file
@ -0,0 +1,10 @@
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /logs/train/*.log
|
||||
Tag app.train
|
||||
Path_Key log_path
|
||||
Refresh_Interval 5
|
||||
DB /buffers/train.db
|
||||
Skip_Long_Lines On
|
||||
storage.type filesystem
|
||||
multiline.parser python,go,java
|
10
src/log/fluent-bit/build/etc/inputs.d/20-infer.conf
Normal file
10
src/log/fluent-bit/build/etc/inputs.d/20-infer.conf
Normal file
@ -0,0 +1,10 @@
|
||||
[INPUT]
|
||||
Name tail
|
||||
Path /logs/infer/*.log
|
||||
Tag app.infer
|
||||
Path_Key log_path
|
||||
Refresh_Interval 5
|
||||
DB /buffers/infer.db
|
||||
Skip_Long_Lines On
|
||||
storage.type filesystem
|
||||
multiline.parser python,go,java
|
24
src/log/fluent-bit/build/etc/outputs.d/10-es.conf
Normal file
24
src/log/fluent-bit/build/etc/outputs.d/10-es.conf
Normal file
@ -0,0 +1,24 @@
|
||||
# 重要:使用 Logstash_Format + Logstash_Prefix,生成 train-*/infer-* 索引
|
||||
[OUTPUT]
|
||||
Name es
|
||||
Match app.train
|
||||
Host ${ES_HOST}
|
||||
Port ${ES_PORT}
|
||||
Logstash_Format On
|
||||
Logstash_Prefix train
|
||||
Replace_Dots On
|
||||
Generate_ID On
|
||||
Retry_Limit False
|
||||
Suppress_Type_Name On
|
||||
|
||||
[OUTPUT]
|
||||
Name es
|
||||
Match app.infer
|
||||
Host ${ES_HOST}
|
||||
Port ${ES_PORT}
|
||||
Logstash_Format On
|
||||
Logstash_Prefix infer
|
||||
Replace_Dots On
|
||||
Generate_ID On
|
||||
Retry_Limit False
|
||||
Suppress_Type_Name On
|
27
src/log/fluent-bit/build/etc/parsers.conf
Normal file
27
src/log/fluent-bit/build/etc/parsers.conf
Normal file
@ -0,0 +1,27 @@
|
||||
[MULTILINE_PARSER]
|
||||
Name python
|
||||
Type regex
|
||||
Flush 2
|
||||
Rule "start_state" "/^\d{4}-\d{2}-\d{2}[\sT]/" "cont"
|
||||
Rule "cont" "/^\s+|^Traceback|^\tat\s+/" "cont"
|
||||
|
||||
[MULTILINE_PARSER]
|
||||
Name go
|
||||
Type regex
|
||||
Flush 2
|
||||
Rule "start_state" "/^[0-9]{4}\/[0-9]{2}\/[0-9]{2}/" "cont"
|
||||
Rule "cont" "/^\s+|^\t/" "cont"
|
||||
|
||||
[MULTILINE_PARSER]
|
||||
Name java
|
||||
Type regex
|
||||
Flush 2
|
||||
Rule "start_state" "/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/" "cont"
|
||||
Rule "cont" "/^\s+at\s+|^\t.../" "cont"
|
||||
|
||||
[PARSER]
|
||||
Name timestamp_parser
|
||||
Format regex
|
||||
Regex ^(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+(?<level>\w+)\s+(?<message>.*)$
|
||||
Time_Key timestamp
|
||||
Time_Format %Y-%m-%d %H:%M:%S
|
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user