NASP/argus
83
0
Fork 0
Code Issues 8 Pull Requests 1 Actions Packages Projects 1 Releases 1 Wiki Activity

dev_1.0.0_sundp_2 优化Argus-metric模块的e2e部署测试流程 (#27)

Browse Source 
Co-authored-by: sundapeng.sdp <sundapeng@hashdata.cn>
Reviewed-on: #27
Reviewed-by: yuyr <yuyr@zgclab.edu.cn>
Reviewed-by: xuxt <xuxt@zgclab.edu.cn>
This commit was merged in pull request #27.
This commit is contained in:
sundapeng
2025-10-17 17:15:55 +08:00
parent 31ccb0b1b8
commit 1a768bc837
36 changed files with 1805 additions and 695 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
src/metric/grafana/build/Dockerfile
View File

@@ -17,30 +17,31 @@ RUN mkdir -p /var/log/supervisor
ENV GRAFANA_BASE_PATH=/private/argus/metric/grafana
# 设置用户和组ID环境变量
ARG GRAFANA_UID=2133
ARG GRAFANA_GID=2015
ENV GRAFANA_UID=${GRAFANA_UID}
ENV GRAFANA_GID=${GRAFANA_GID}
ARG ARGUS_BUILD_UID=2133
ARG ARGUS_BUILD_GID=2015
ENV ARGUS_BUILD_UID=${ARGUS_BUILD_UID} \
ARGUS_BUILD_GID=${ARGUS_BUILD_GID}
# 创建基本目录结构
RUN mkdir -p /private/argus/etc \
&& mkdir -p /private/argus/metric/grafana/data \
&& mkdir -p /private/argus/metric/grafana/logs \
&& mkdir -p /private/argus/metric/grafana/plugins \
&& mkdir -p /private/argus/metric/grafana/provisioning/datasources \
&& mkdir -p /private/argus/metric/grafana/provisioning/dashboards \
&& mkdir -p /private/argus/metric/grafana/data/sessions \
&& mkdir -p /private/argus/metric/grafana/data/dashboards \
&& mkdir -p /private/argus/metric/grafana/config \
&& mkdir -p ${GRAFANA_BASE_PATH}/data \
&& mkdir -p ${GRAFANA_BASE_PATH}/logs \
&& mkdir -p ${GRAFANA_BASE_PATH}/plugins \
&& mkdir -p ${GRAFANA_BASE_PATH}/provisioning/datasources \
&& mkdir -p ${GRAFANA_BASE_PATH}/provisioning/dashboards \
&& mkdir -p ${GRAFANA_BASE_PATH}/data/sessions \
&& mkdir -p ${GRAFANA_BASE_PATH}/data/dashboards \
&& mkdir -p ${GRAFANA_BASE_PATH}/config \
&& mkdir -p /etc/grafana \
&& mkdir -p /var/lib/grafana \
&& mkdir -p /var/log/grafana
# 修改 Grafana 用户 UID/GID 并授权
RUN deluser grafana && \
addgroup -g ${GRAFANA_GID} grafana && \
adduser -u ${GRAFANA_UID} -G grafana -s /bin/sh -D grafana && \
chown -R grafana:grafana /var/lib/grafana /etc/grafana /var/log/grafana /private/argus
addgroup -g ${ARGUS_BUILD_GID} grafana && \
adduser -u ${ARGUS_BUILD_UID} -G grafana -s /bin/sh -D grafana && \
chown -R grafana:grafana /var/lib/grafana /etc/grafana /var/log/grafana ${GRAFANA_BASE_PATH}
# 复制配置文件到容器内临时位置
COPY grafana.ini /tmp/grafana.ini

570
src/metric/grafana/build/dashboards/default_cluster_dashboard.json Normal file
View File

@@ -0,0 +1,570 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 3,
"links": [],
"panels": [
{
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 0,
"y": 0
},
"id": 1,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "100 * (1 - avg(rate(node_cpu_seconds_total{mode='idle'}[5m])))",
"refId": "A"
}
],
"title": "CPU 平均利用率（%）",
"type": "stat"
},
{
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 8,
"y": 0
},
"id": 2,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"mean"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "avg(1 - ((node_memory_MemFree_bytes + node_memory_Buffers_bytes + node_memory_Cached_bytes) / node_memory_MemTotal_bytes)) * 100",
"refId": "A"
}
],
"title": "内存平均利用率（%）",
"type": "stat"
},
{
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 5,
"w": 8,
"x": 16,
"y": 0
},
"id": 4,
"options": {
"colorMode": "value",
"graphMode": "none",
"justifyMode": "auto",
"orientation": "auto",
"percentChangeColorMode": "standard",
"reduceOptions": {
"calcs": [
"last"
],
"fields": "",
"values": false
},
"showPercentChange": false,
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "count(count by(hostname) (up{job='node'} == 1))",
"refId": "A"
}
],
"title": "节点在线数",
"type": "stat"
},
{
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 0,
"y": 5
},
"id": 6,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto"
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "avg by (hostname) (DCGM_FI_DEV_GPU_UTIL)",
"refId": "A"
}
],
"title": "GPU 平均利用率 (%)",
"type": "gauge"
},
{
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 6,
"y": 5
},
"id": 12,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto"
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "round(avg(DCGM_FI_DEV_FB_USED{job='dcgm'}/(DCGM_FI_DEV_FB_USED{job='dcgm'} + DCGM_FI_DEV_FB_FREE{job='dcgm'})) * 100)",
"refId": "A"
}
],
"title": "显存平均利用率 (%)",
"type": "gauge"
},
{
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"mappings": [],
"max": 100,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 12,
"y": 5
},
"id": 7,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": false,
"showThresholdMarkers": true,
"sizing": "auto"
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "avg by (hostname) (DCGM_FI_DEV_GPU_TEMP)",
"refId": "A"
}
],
"title": "GPU 温度 (℃)",
"type": "gauge"
},
{
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"mappings": [],
"max": 300,
"min": 0,
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "orange",
"value": 200
},
{
"color": "red",
"value": 300
}
]
},
"unit": "watt"
},
"overrides": []
},
"gridPos": {
"h": 6,
"w": 6,
"x": 18,
"y": 5
},
"id": 8,
"options": {
"minVizHeight": 75,
"minVizWidth": 75,
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"showThresholdLabels": true,
"showThresholdMarkers": true,
"sizing": "auto"
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "avg by (hostname) (DCGM_FI_DEV_POWER_USAGE)",
"refId": "A"
}
],
"title": "GPU 平均实时功耗 (W)",
"type": "gauge"
},
{
"datasource": "prometheus",
"fieldConfig": {
"defaults": {
"custom": {
"align": "center",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"decimals": 1,
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"unit": "percent"
},
"overrides": []
},
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 11
},
"id": 11,
"options": {
"cellHeight": "sm",
"cellLinks": [
{
"title": "跳转至节点详情",
"url": "http://127.0.0.1:3000/d/node_gpu_metrics/node-and-gpu-metrics?orgId=1&refresh=15s&var-hostname=${__data.fields.hostname}"
}
],
"footer": {
"countRows": false,
"fields": "",
"reducer": [
"sum"
],
"show": false
},
"showHeader": true,
"sortBy": [
{
"desc": true,
"displayName": "GPU 使用率"
}
]
},
"pluginVersion": "11.1.0",
"targets": [
{
"expr": "up{job=\"dcgm\"} + on(hostname) group_left(ip, node_id) up{job=\"dcgm\"}*0",
"format": "table",
"instant": true,
"refId": "node_info"
},
{
"expr": "round(100 - avg by(hostname)(rate(node_cpu_seconds_total{job=\"node\",mode=\"idle\"}[5m])) * 100, 0.1)",
"format": "table",
"instant": true,
"refId": "CPU"
},
{
"expr": "round((1 - avg by(hostname)(node_memory_MemAvailable_bytes{job=\"node\"} / node_memory_MemTotal_bytes{job=\"node\"})) * 100, 0.1)",
"format": "table",
"instant": true,
"refId": "MEM"
},
{
"expr": "round(avg by(hostname)(DCGM_FI_DEV_GPU_UTIL{job=\"dcgm\"}), 0.1)",
"format": "table",
"instant": true,
"refId": "GPU_UTIL"
},
{
"expr": "round(avg by(hostname)(DCGM_FI_DEV_FB_USED{job=\"dcgm\"} / (DCGM_FI_DEV_FB_USED{job=\"dcgm\"} + DCGM_FI_DEV_FB_FREE{job=\"dcgm\"}) * 100), 0.1)",
"format": "table",
"instant": true,
"refId": "GPU_MEM"
}
],
"title": "节点列表（CPU / 内存 / GPU）",
"transformations": [
{
"id": "seriesToColumns",
"options": {
"byField": "hostname"
}
},
{
"id": "organize",
"options": {
"excludeByName": {
"Time": true,
"Value #node_info": true,
"hostname_1": true,
"hostname_2": true,
"hostname_3": true,
"instance": true,
"ip_1": true,
"job": true,
"node_id_1": true
},
"indexByName": {
"CPU 使用率": 3,
"GPU 使用率": 5,
"GPU 显存占用": 6,
"IP 地址": 1,
"主机名": 0,
"内存使用率": 4,
"节点 ID": 2
},
"renameByName": {
"Value #CPU": "CPU 使用率",
"Value #GPU_MEM": "GPU 显存占用",
"Value #GPU_UTIL": "GPU 使用率",
"Value #MEM": "内存使用率",
"hostname": "主机名",
"ip": "IP 地址",
"node_id": "节点 ID",
"user_id": "用户ID"
}
}
}
],
"type": "table"
}
],
"refresh": "5s",
"schemaVersion": 39,
"tags": [
"cluster",
"gpu",
"system"
],
"templating": {
"list": []
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {},
"timezone": "browser",
"title": "Cluster Dashboard",
"uid": "cluster-dashboard",
"version": 34,
"weekStart": ""
}

10
src/metric/grafana/build/start-grafana-supervised.sh
View File

@@ -9,6 +9,7 @@ DOMAIN=grafana.metric.argus.com
IP=$(ifconfig | awk '/inet / && $2 != "127.0.0.1" {print $2; exit}')
echo "current IP: ${IP}"
echo "${IP}" > /private/argus/etc/${DOMAIN}
chmod +x /private/argus/etc/${DOMAIN}
# 确保必要目录存在（权限已在 Dockerfile 中设置）
mkdir -p /private/argus/metric/grafana/data
@@ -27,7 +28,6 @@ mkdir -p /var/lib/grafana
if [ -f "/tmp/grafana.ini" ]; then
echo "[INFO] Copying grafana.ini to /private/argus/metric/grafana/config/"
cp /tmp/grafana.ini /private/argus/metric/grafana/config/grafana.ini
chown grafana:grafana /private/argus/metric/grafana/config/grafana.ini
echo "[INFO] Grafana configuration copied successfully"
fi
@@ -47,12 +47,9 @@ fi
if [ -f "/tmp/datasources.yml" ]; then
echo "[INFO] Copying datasource configuration to /private/argus/metric/grafana/provisioning/datasources/"
cp /tmp/datasources.yml /private/argus/metric/grafana/provisioning/datasources/datasources.yml
chown grafana:grafana /private/argus/metric/grafana/provisioning/datasources/datasources.yml
echo "[INFO] Datasource configuration copied successfully"
elif [ -d "/private/argus/metric/grafana/provisioning/datasources" ] && [ "$(ls -A /private/argus/metric/grafana/provisioning/datasources)" ]; then
echo "[INFO] Found existing datasource provisioning files in /private/argus/metric/grafana/provisioning/datasources"
# 确保数据源配置目录权限正确
chown -R grafana:grafana /private/argus/metric/grafana/provisioning/datasources
elif [ -d "/etc/grafana/provisioning/datasources" ] && [ "$(ls -A /etc/grafana/provisioning/datasources)" ]; then
echo "[INFO] Found datasource provisioning files in /etc/grafana/provisioning/datasources"
# 确保数据源配置目录权限正确
@@ -65,7 +62,6 @@ fi
if [ -f "/tmp/dashboards.yml" ]; then
echo "[INFO] Copying dashboard configuration to /private/argus/metric/grafana/provisioning/dashboards/"
cp /tmp/dashboards.yml /private/argus/metric/grafana/provisioning/dashboards/dashboards.yml
chown grafana:grafana /private/argus/metric/grafana/provisioning/dashboards/dashboards.yml
echo "[INFO] Dashboard configuration copied successfully"
fi
@@ -73,13 +69,9 @@ fi
if [ -f "/tmp/default_dashboard.json" ]; then
echo "[INFO] Copying default dashboard to /private/argus/metric/grafana/provisioning/dashboards/"
cp /tmp/default_dashboard.json /private/argus/metric/grafana/provisioning/dashboards/default_dashboard.json
chown grafana:grafana /private/argus/metric/grafana/provisioning/dashboards/default_dashboard.json
echo "[INFO] Default dashboard copied successfully"
fi
# 确保所有配置目录权限正确
chown -R grafana:grafana /private/argus/metric/grafana/provisioning/
# 启动 Grafana
if [ -n "$CONFIG_FILE" ]; then
echo "[INFO] Starting Grafana with custom configuration..."