#!/usr/bin/env bash set -euo pipefail TMP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)/tmp/metric-verify" mkdir -p "$TMP_DIR" # 载入端口变量(由 .env 提供) TEST_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)" if [[ -f "$TEST_ROOT/.env" ]]; then set -a; source "$TEST_ROOT/.env"; set +a fi PROM_BASE="http://localhost:${PROMETHEUS_PORT:-9090}/api/v1" INSTANCE="${METRIC_TEST_INSTANCE:-172.31.0.50:9100}" IP_ONLY="${INSTANCE%%:*}" echo "[VERIFY:DATA] node exporter metrics present in container" docker exec argus-metric-test-node bash -lc "curl -fsS --max-time 5 http://localhost:9100/metrics | head -n 5" > "$TMP_DIR/node_metrics_head.txt" || { echo "[ERR] cannot fetch node exporter metrics" >&2; exit 1; } if ! grep -E "node_(exporter_build_info|time_seconds)" -q "$TMP_DIR/node_metrics_head.txt"; then echo "[WARN] head did not show expected lines; continuing (exporter may output later lines)" fi echo "[OK] node exporter endpoint reachable" echo "[VERIFY:DATA] Prometheus has recent sample for build_info" curl -fsS --max-time 5 --get "$PROM_BASE/query" --data-urlencode "query=node_exporter_build_info{job=\"node\",ip=\"$IP_ONLY\"}" > "$TMP_DIR/prom_ne_build_info_1.json" python3 - "$TMP_DIR/prom_ne_build_info_1.json" <<'PY' import json,sys,time j=json.load(open(sys.argv[1])) res=j.get('data',{}).get('result',[]) assert res, 'no result for node_exporter_build_info' ts=float(res[0]['value'][0]) now=time.time() assert now-ts<180, f"sample too old: now={now} ts={ts}" print(int(ts)) PY T1=$? sleep 30 curl -fsS --max-time 5 --get "$PROM_BASE/query" --data-urlencode "query=node_exporter_build_info{job=\"node\",ip=\"$IP_ONLY\"}" > "$TMP_DIR/prom_ne_build_info_2.json" TS1=$(python3 - "$TMP_DIR/prom_ne_build_info_1.json" <<'PY' import json,sys print(float(json.load(open(sys.argv[1]))['data']['result'][0]['value'][0])) PY ) TS2=$(python3 - "$TMP_DIR/prom_ne_build_info_2.json" <<'PY' import json,sys print(float(json.load(open(sys.argv[1]))['data']['result'][0]['value'][0])) PY ) awk -v a="$TS1" -v b="$TS2" 'BEGIN{ if (b>=a) exit 0; else exit 1 }' || { echo "[ERR] sample timestamp did not advance" >&2; exit 1; } echo "[OK] sample timestamp advanced" echo "[DONE] dataplane verify" # 追加:GPU 节点端点连通性检查(启用 GPU 时) if [[ "${ENABLE_GPU:-false}" == "true" ]]; then echo echo "[VERIFY:DATA][GPU] curl endpoints on gpu node" if ! docker exec argus-metric-test-gpu-node bash -lc 'curl -fsS --max-time 5 http://localhost:9100/metrics >/dev/null'; then echo "[ERR] gpu node 9100 not reachable" >&2; exit 1 fi if ! docker exec argus-metric-test-gpu-node bash -lc 'curl -fsS --max-time 5 http://localhost:9400/metrics >/dev/null'; then echo "[ERR] gpu node 9400 not reachable" >&2; exit 1 fi echo "[OK] gpu node endpoints reachable" fi