src/log/fluent-bit/build/etc/fluent-bit.conf

@@ -0,0 +1,37 @@
+[SERVICE]
+    Daemon       Off
+    Parsers_File parsers.conf
+    HTTP_Server  On
+    HTTP_Listen  0.0.0.0
+    HTTP_Port    2020
+    storage.path /buffers
+    storage.sync normal
+    storage.checksum on
+    storage.backlog.mem_limit 128M
+    # 备注：该镜像默认未开启 Hot Reload，修改配置后请重启容器。
+
+@INCLUDE inputs.d/*.conf
+
+[FILTER]
+    Name   parser
+    Match  app.*
+    Key_Name log
+    Parser timestamp_parser
+    Reserve_Data On
+    Preserve_Key On
+    Unescape_Key On
+
+[FILTER]
+    Name   record_modifier
+    Match  *
+    Record cluster  ${CLUSTER}
+    Record rack     ${RACK}
+    Record host     ${HOSTNAME}
+
+[FILTER]
+    Name   lua
+    Match  app.*
+    script inject_labels.lua
+    call   add_labels
+
+@INCLUDE outputs.d/*.conf

src/log/fluent-bit/build/etc/inject_labels.lua

@@ -0,0 +1,15 @@
+function add_labels(tag, ts, record)
+  record["job_id"] = os.getenv("FB_JOB_ID") or record["job_id"] or "unknown"
+  record["user"]   = os.getenv("FB_USER")   or record["user"]   or "unknown"
+  record["model"]  = os.getenv("FB_MODEL")  or record["model"]  or "unknown"
+  record["gpu_id"] = os.getenv("FB_GPU_ID") or record["gpu_id"] or "na"
+  local p = record["log_path"] or ""
+  if string.find(p, "/logs/infer/") then
+    record["role"] = "infer"
+  elseif string.find(p, "/logs/train/") then
+    record["role"] = "train"
+  else
+    record["role"] = record["role"] or "app"
+  end
+  return 1, ts, record
+end

src/log/fluent-bit/build/etc/inputs.d/10-train.conf

@@ -0,0 +1,10 @@
+[INPUT]
+    Name              tail
+    Path              /logs/train/*.log
+    Tag               app.train
+    Path_Key          log_path
+    Refresh_Interval  5
+    DB                /buffers/train.db
+    Skip_Long_Lines   On
+    storage.type      filesystem
+    multiline.parser  python,go,java

src/log/fluent-bit/build/etc/inputs.d/20-infer.conf

@@ -0,0 +1,10 @@
+[INPUT]
+    Name              tail
+    Path              /logs/infer/*.log
+    Tag               app.infer
+    Path_Key          log_path
+    Refresh_Interval  5
+    DB                /buffers/infer.db
+    Skip_Long_Lines   On
+    storage.type      filesystem
+    multiline.parser  python,go,java

src/log/fluent-bit/build/etc/outputs.d/10-es.conf

@@ -0,0 +1,24 @@
+# 重要：使用 Logstash_Format + Logstash_Prefix，生成 train-*/infer-* 索引
+[OUTPUT]
+    Name                es
+    Match               app.train
+    Host                ${ES_HOST}
+    Port                ${ES_PORT}
+    Logstash_Format     On
+    Logstash_Prefix     train
+    Replace_Dots        On
+    Generate_ID         On
+    Retry_Limit         False
+    Suppress_Type_Name  On
+
+[OUTPUT]
+    Name                es
+    Match               app.infer
+    Host                ${ES_HOST}
+    Port                ${ES_PORT}
+    Logstash_Format     On
+    Logstash_Prefix     infer
+    Replace_Dots        On
+    Generate_ID         On
+    Retry_Limit         False
+    Suppress_Type_Name  On

src/log/fluent-bit/build/etc/parsers.conf

@@ -0,0 +1,27 @@
+[MULTILINE_PARSER]
+    Name   python
+    Type   regex
+    Flush  2
+    Rule   "start_state"  "/^\d{4}-\d{2}-\d{2}[\sT]/"  "cont"
+    Rule   "cont"         "/^\s+|^Traceback|^\tat\s+/" "cont"
+
+[MULTILINE_PARSER]
+    Name   go
+    Type   regex
+    Flush  2
+    Rule   "start_state"  "/^[0-9]{4}\/[0-9]{2}\/[0-9]{2}/" "cont"
+    Rule   "cont"         "/^\s+|^\t/" "cont"
+
+[MULTILINE_PARSER]
+    Name   java
+    Type   regex
+    Flush  2
+    Rule   "start_state"  "/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/" "cont"
+    Rule   "cont"         "/^\s+at\s+|^\t.../" "cont"
+
+[PARSER]
+    Name   timestamp_parser
+    Format regex
+    Regex  ^(?<timestamp>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+(?<level>\w+)\s+(?<message>.*)$
+    Time_Key    timestamp
+    Time_Format %Y-%m-%d %H:%M:%S