[#37] 增加sys/swarm_tests(cpu) ；单独构建的node bundle镜像
This commit is contained in:
103
deployment/build/templates/scripts/fix-prom-targets-overlay.sh
Normal file
103
deployment/build/templates/scripts/fix-prom-targets-overlay.sh
Normal file
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Quick fix tool: replace 172.22/16 targets in nodes.json with overlay IPs resolved from hostname.
|
||||
# Usage: run on server package host: scripts/fix-prom-targets-overlay.sh
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
NODES_JSON="$ROOT/private/argus/metric/prometheus/nodes.json"
|
||||
|
||||
require_cmd() { command -v "$1" >/dev/null 2>&1 || { echo "[ERROR] missing command: $1" >&2; exit 1; }; }
|
||||
|
||||
backup() {
|
||||
local src="$1"; local ts; ts=$(date -u +%Y%m%d-%H%M%SZ)
|
||||
cp "$src" "${src%.json}_bak_${ts}.json"
|
||||
}
|
||||
|
||||
prefer_overlay_ip() {
|
||||
local host="$1"
|
||||
# prefer 10.0/8 then 172.31/16
|
||||
getent hosts "$host" | awk '{print $1}' | while read -r ip; do
|
||||
if [[ "$ip" =~ ^10\. ]]; then echo "$ip"; return; fi
|
||||
done
|
||||
getent hosts "$host" | awk '{print $1}' | while read -r ip; do
|
||||
if [[ "$ip" =~ ^172\.31\. ]]; then echo "$ip"; return; fi
|
||||
done
|
||||
# fallback: first A record
|
||||
getent hosts "$host" | awk '{print $1; exit}'
|
||||
}
|
||||
|
||||
require_cmd awk
|
||||
require_cmd sed
|
||||
|
||||
if [[ ! -f "$NODES_JSON" ]]; then
|
||||
echo "[WARN] nodes.json not found: $NODES_JSON" >&2
|
||||
exit 0
|
||||
fi
|
||||
|
||||
backup "$NODES_JSON"
|
||||
|
||||
tmp=$(mktemp)
|
||||
trap 'rm -f "$tmp"' EXIT
|
||||
|
||||
changed=0
|
||||
python3 - "$NODES_JSON" <<'PY' > "$tmp" || {
|
||||
import ipaddress, json, sys, socket
|
||||
path=sys.argv[1]
|
||||
data=json.load(open(path)) if path else []
|
||||
def resolve(host):
|
||||
try:
|
||||
infos=socket.getaddrinfo(host,None,family=socket.AF_INET)
|
||||
ips=[i[4][0] for i in infos]
|
||||
# prefer 10. over 172.31.
|
||||
for ip in ips:
|
||||
if ip.startswith('10.'): return ip
|
||||
for ip in ips:
|
||||
if ip.startswith('172.31.'): return ip
|
||||
return ips[0] if ips else None
|
||||
except OSError:
|
||||
return None
|
||||
gw=ipaddress.ip_network('172.22.0.0/16')
|
||||
out=[]
|
||||
changed=False
|
||||
for item in data:
|
||||
ip=item.get('ip')
|
||||
host=item.get('hostname') or ''
|
||||
try:
|
||||
bad = ip and ipaddress.ip_address(ip) in gw
|
||||
except Exception:
|
||||
bad = False
|
||||
if bad and host:
|
||||
new=resolve(host)
|
||||
if new:
|
||||
item=dict(item)
|
||||
item['ip']=new
|
||||
changed=True
|
||||
out.append(item)
|
||||
json.dump(out, sys.stdout, ensure_ascii=False)
|
||||
sys.stderr.write('CHANGED' if changed else 'UNCHANGED')
|
||||
PY
|
||||
|
||||
status=$?
|
||||
marker=$(tail -n1 /dev/stderr 2>/dev/null || true)
|
||||
if [[ "$status" -ne 0 ]]; then
|
||||
echo "[ERROR] failed to rewrite nodes.json" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if grep -q '"ip"\s*:\s*"172\.22\.' "$tmp"; then
|
||||
echo "[WARN] some gwbridge targets remain; manual fix may be required" >&2
|
||||
fi
|
||||
|
||||
mv "$tmp" "$NODES_JSON"
|
||||
echo "[OK] nodes.json updated"
|
||||
|
||||
# try to reload Prometheus
|
||||
if docker ps --format '{{.Names}}' | grep -q '^argus-prometheus$'; then
|
||||
docker exec argus-prometheus sh -lc 'pidof prometheus >/dev/null 2>&1 && kill -HUP $(pidof prometheus) || supervisorctl restart prometheus' >/dev/null 2>&1 || true
|
||||
echo "[INFO] Prometheus reloaded"
|
||||
fi
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -155,6 +155,34 @@ gfcode=$(http_code "http://localhost:${GRAFANA_PORT:-3000}/api/health"); [[ "$gf
|
||||
# Deduplicate errors
|
||||
sort -u -o "$ERRORS" "$ERRORS"
|
||||
|
||||
# --- Prometheus targets & nodes.json checks ---
|
||||
section PROMETHEUS-TARGETS
|
||||
nodes_json_path="$ROOT/private/argus/metric/prometheus/nodes.json"
|
||||
if [[ -f "$nodes_json_path" ]]; then
|
||||
logd "nodes.json present: $nodes_json_path"
|
||||
# detect gwbridge addresses (172.22/16)
|
||||
if grep -E '"ip"\s*:\s*"172\.22\.' "$nodes_json_path" >/dev/null 2>&1; then
|
||||
append_err "[prometheus][targets] contains gwbridge (172.22/16) addresses; prefer overlay (10.0/8)."
|
||||
echo "HINT: nodes.json has 172.22.x.x targets. Ensure agent publishes overlay_ip and master prefers it." >&2
|
||||
fi
|
||||
else
|
||||
logd "nodes.json missing at $nodes_json_path"
|
||||
fi
|
||||
|
||||
# Query Prometheus activeTargets and list down items when possible
|
||||
pt_json=$(curl -s --max-time 3 "http://localhost:${PROMETHEUS_PORT:-9090}/api/v1/targets" || true)
|
||||
if command -v jq >/dev/null 2>&1; then
|
||||
downs=$(printf '%s' "$pt_json" | jq -r '.data.activeTargets[] | select(.health=="down") | "[prometheus][activeTargets] down url=\(.scrapeUrl) lastError=\(.lastError)"' 2>/dev/null || true)
|
||||
if [[ -n "$downs" ]]; then
|
||||
printf '%s\n' "$downs" >> "$ERRORS"
|
||||
fi
|
||||
else
|
||||
# best-effort grep when jq is unavailable
|
||||
if printf '%s' "$pt_json" | grep -q '"health":"down"'; then
|
||||
append_err "[prometheus][activeTargets] some targets are down (enable jq for detailed reasons)"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo "Diagnostic details -> $DETAILS"
|
||||
echo "Detected errors -> $ERRORS"
|
||||
|
||||
|
||||
Reference in New Issue
Block a user