diff --git a/src/log/fluent-bit/build/etc/fluent-bit.conf b/src/log/fluent-bit/build/etc/fluent-bit.conf new file mode 100644 index 0000000..95ed374 --- /dev/null +++ b/src/log/fluent-bit/build/etc/fluent-bit.conf @@ -0,0 +1,37 @@ +[SERVICE] + Daemon Off + Parsers_File parsers.conf + HTTP_Server On + HTTP_Listen 0.0.0.0 + HTTP_Port 2020 + storage.path /buffers + storage.sync normal + storage.checksum on + storage.backlog.mem_limit 128M + # 备注:该镜像默认未开启 Hot Reload,修改配置后请重启容器。 + +@INCLUDE inputs.d/*.conf + +[FILTER] + Name parser + Match app.* + Key_Name log + Parser timestamp_parser + Reserve_Data On + Preserve_Key On + Unescape_Key On + +[FILTER] + Name record_modifier + Match * + Record cluster ${CLUSTER} + Record rack ${RACK} + Record host ${HOSTNAME} + +[FILTER] + Name lua + Match app.* + script inject_labels.lua + call add_labels + +@INCLUDE outputs.d/*.conf diff --git a/src/log/fluent-bit/build/etc/inject_labels.lua b/src/log/fluent-bit/build/etc/inject_labels.lua new file mode 100644 index 0000000..0d87f7a --- /dev/null +++ b/src/log/fluent-bit/build/etc/inject_labels.lua @@ -0,0 +1,15 @@ +function add_labels(tag, ts, record) + record["job_id"] = os.getenv("FB_JOB_ID") or record["job_id"] or "unknown" + record["user"] = os.getenv("FB_USER") or record["user"] or "unknown" + record["model"] = os.getenv("FB_MODEL") or record["model"] or "unknown" + record["gpu_id"] = os.getenv("FB_GPU_ID") or record["gpu_id"] or "na" + local p = record["log_path"] or "" + if string.find(p, "/logs/infer/") then + record["role"] = "infer" + elseif string.find(p, "/logs/train/") then + record["role"] = "train" + else + record["role"] = record["role"] or "app" + end + return 1, ts, record +end diff --git a/src/log/fluent-bit/build/etc/inputs.d/10-train.conf b/src/log/fluent-bit/build/etc/inputs.d/10-train.conf new file mode 100644 index 0000000..3ea9e25 --- /dev/null +++ b/src/log/fluent-bit/build/etc/inputs.d/10-train.conf @@ -0,0 +1,10 @@ +[INPUT] + Name tail + Path /logs/train/*.log + Tag app.train + Path_Key log_path + Refresh_Interval 5 + DB /buffers/train.db + Skip_Long_Lines On + storage.type filesystem + multiline.parser python,go,java diff --git a/src/log/fluent-bit/build/etc/inputs.d/20-infer.conf b/src/log/fluent-bit/build/etc/inputs.d/20-infer.conf new file mode 100644 index 0000000..793e203 --- /dev/null +++ b/src/log/fluent-bit/build/etc/inputs.d/20-infer.conf @@ -0,0 +1,10 @@ +[INPUT] + Name tail + Path /logs/infer/*.log + Tag app.infer + Path_Key log_path + Refresh_Interval 5 + DB /buffers/infer.db + Skip_Long_Lines On + storage.type filesystem + multiline.parser python,go,java diff --git a/src/log/fluent-bit/build/etc/outputs.d/10-es.conf b/src/log/fluent-bit/build/etc/outputs.d/10-es.conf new file mode 100644 index 0000000..eea46fd --- /dev/null +++ b/src/log/fluent-bit/build/etc/outputs.d/10-es.conf @@ -0,0 +1,24 @@ +# 重要:使用 Logstash_Format + Logstash_Prefix,生成 train-*/infer-* 索引 +[OUTPUT] + Name es + Match app.train + Host ${ES_HOST} + Port ${ES_PORT} + Logstash_Format On + Logstash_Prefix train + Replace_Dots On + Generate_ID On + Retry_Limit False + Suppress_Type_Name On + +[OUTPUT] + Name es + Match app.infer + Host ${ES_HOST} + Port ${ES_PORT} + Logstash_Format On + Logstash_Prefix infer + Replace_Dots On + Generate_ID On + Retry_Limit False + Suppress_Type_Name On diff --git a/src/log/fluent-bit/build/etc/parsers.conf b/src/log/fluent-bit/build/etc/parsers.conf new file mode 100644 index 0000000..d86fa06 --- /dev/null +++ b/src/log/fluent-bit/build/etc/parsers.conf @@ -0,0 +1,27 @@ +[MULTILINE_PARSER] + Name python + Type regex + Flush 2 + Rule "start_state" "/^\d{4}-\d{2}-\d{2}[\sT]/" "cont" + Rule "cont" "/^\s+|^Traceback|^\tat\s+/" "cont" + +[MULTILINE_PARSER] + Name go + Type regex + Flush 2 + Rule "start_state" "/^[0-9]{4}\/[0-9]{2}\/[0-9]{2}/" "cont" + Rule "cont" "/^\s+|^\t/" "cont" + +[MULTILINE_PARSER] + Name java + Type regex + Flush 2 + Rule "start_state" "/^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}/" "cont" + Rule "cont" "/^\s+at\s+|^\t.../" "cont" + +[PARSER] + Name timestamp_parser + Format regex + Regex ^(?\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2})\s+(?\w+)\s+(?.*)$ + Time_Key timestamp + Time_Format %Y-%m-%d %H:%M:%S diff --git a/src/log/fluent-bit/fluent-bit-bundle.tar.gz b/src/log/fluent-bit/build/packages/fluent-bit_3.1.9_amd64.deb similarity index 81% rename from src/log/fluent-bit/fluent-bit-bundle.tar.gz rename to src/log/fluent-bit/build/packages/fluent-bit_3.1.9_amd64.deb index e46b153..2b1f68f 100644 Binary files a/src/log/fluent-bit/fluent-bit-bundle.tar.gz and b/src/log/fluent-bit/build/packages/fluent-bit_3.1.9_amd64.deb differ diff --git a/src/log/fluent-bit/start-fluent-bit.sh b/src/log/fluent-bit/build/start-fluent-bit.sh similarity index 100% rename from src/log/fluent-bit/start-fluent-bit.sh rename to src/log/fluent-bit/build/start-fluent-bit.sh