[#2] master模块开发测试

2025-09-24 03:48:05 +00:00
parent 5acc1fe92b
commit d9461b3991
29 changed files with 1907 additions and 0 deletions
--- a/src/master/tests/.gitignore
+++ b/src/master/tests/.gitignore
@@ -0,0 +1,2 @@
+private/
+tmp/
--- a/src/master/tests/docker-compose.yml
+++ b/src/master/tests/docker-compose.yml
@@ -0,0 +1,18 @@
+services:
+  master:
+    image: argus-master:dev
+    container_name: argus-master-e2e
+    environment:
+      - OFFLINE_THRESHOLD_SECONDS=6
+      - ONLINE_THRESHOLD_SECONDS=2
+      - SCHEDULER_INTERVAL_SECONDS=1
+    ports:
+      - "31300:3000"
+    volumes:
+      - ./private/argus/master:/private/argus/master
+      - ./private/argus/metric/prometheus:/private/argus/metric/prometheus
+    restart: unless-stopped
+
+networks:
+  default:
+    driver: bridge
--- a/src/master/tests/scripts/00_e2e_test.sh
+++ b/src/master/tests/scripts/00_e2e_test.sh
@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTS=(
+  "01_up_master.sh"
+  "02_verify_ready_and_nodes_json.sh"
+  "03_register_via_curl.sh"
+  "04_reregister_and_error_cases.sh"
+  "05_status_report_via_curl.sh"
+  "06_config_update_and_nodes_json.sh"
+  "07_stats_single_node.sh"
+  "08_multi_node_stats.sh"
+  "09_restart_persistence.sh"
+  "10_down.sh"
+)
+
+for script in "${SCRIPTS[@]}"; do
+  echo "[TEST] Running $script"
+  "$SCRIPT_DIR/$script"
+  echo "[TEST] $script completed"
+  echo
+done
+
+echo "[TEST] Master module E2E tests completed"
--- a/src/master/tests/scripts/01_up_master.sh
+++ b/src/master/tests/scripts/01_up_master.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+MODULE_ROOT="$(cd "$TEST_ROOT/.." && pwd)"
+PRIVATE_ROOT="$TEST_ROOT/private"
+TMP_ROOT="$TEST_ROOT/tmp"
+
+mkdir -p "$PRIVATE_ROOT/argus/master"
+mkdir -p "$PRIVATE_ROOT/argus/metric/prometheus"
+mkdir -p "$TMP_ROOT"
+
+# 确保上一次运行留下的容器/数据被清理
+compose() {
+  if docker compose version >/dev/null 2>&1; then
+    docker compose "$@"
+  else
+    docker-compose "$@"
+  fi
+}
+
+pushd "$TEST_ROOT" >/dev/null
+compose down --remove-orphans || true
+popd >/dev/null
+
+rm -rf "$TMP_ROOT" "$PRIVATE_ROOT"
+mkdir -p "$PRIVATE_ROOT/argus/master"
+mkdir -p "$PRIVATE_ROOT/argus/metric/prometheus"
+mkdir -p "$TMP_ROOT"
+
+pushd "$MODULE_ROOT" >/dev/null
+./scripts/build_images.sh --tag argus-master:dev
+popd >/dev/null
+
+pushd "$TEST_ROOT" >/dev/null
+compose down --remove-orphans || true
+compose up -d
+popd >/dev/null
+
+echo "[INFO] Master container is up on http://localhost:31300"
--- a/src/master/tests/scripts/02_verify_ready_and_nodes_json.sh
+++ b/src/master/tests/scripts/02_verify_ready_and_nodes_json.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+PRIVATE_ROOT="$TEST_ROOT/private"
+API_BASE="http://localhost:31300"
+NODES_JSON_PATH="$PRIVATE_ROOT/argus/metric/prometheus/nodes.json"
+
+# 等待 readyz 返回 200，确保数据库初始化完成
+for _ in {1..30}; do
+  status=$(curl -s -o /dev/null -w '%{http_code}' "$API_BASE/readyz" || true)
+  if [[ "$status" == "200" ]]; then
+    break
+  fi
+  sleep 1
+ done
+
+if [[ "${status:-}" != "200" ]]; then
+  echo "[ERROR] /readyz 未在预期时间内返回 200，实际=$status" >&2
+  exit 1
+fi
+
+echo "[INFO] /readyz 已通过，就绪检查成功"
+
+# scheduler 启动时会产生空的 nodes.json，这里等待文件出现并校验内容
+for _ in {1..30}; do
+  if [[ -f "$NODES_JSON_PATH" ]]; then
+    break
+  fi
+  sleep 1
+ done
+
+if [[ ! -f "$NODES_JSON_PATH" ]]; then
+  echo "[ERROR] 未在预期时间内生成 $NODES_JSON_PATH" >&2
+  exit 1
+fi
+
+if ! python3 - "$NODES_JSON_PATH" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    data = json.load(handle)
+if data != []:
+    raise SystemExit(f"nodes.json initial content should be [], got {data}")
+PY
+then
+  echo "[ERROR] nodes.json 初始内容不是空数组" >&2
+  exit 1
+fi
+
+echo "[INFO] nodes.json 初始状态校验通过"
--- a/src/master/tests/scripts/03_register_via_curl.sh
+++ b/src/master/tests/scripts/03_register_via_curl.sh
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+TMP_ROOT="$TEST_ROOT/tmp"
+API_BASE="http://localhost:31300/api/v1/master"
+
+mkdir -p "$TMP_ROOT"
+
+for _ in {1..30}; do
+  if curl -sf "$API_BASE/healthz" >/dev/null; then
+    break
+  fi
+  sleep 1
+done
+
+payload=$(cat <<'JSON'
+{
+  "name": "dev-testuser-testinst-pod-0",
+  "type": "agent",
+  "meta_data": {
+    "hostname": "dev-testuser-testinst-pod-0",
+    "ip": "10.0.0.10",
+    "env": "dev",
+    "user": "testuser",
+    "instance": "testinst",
+    "cpu_number": 4,
+    "memory_in_bytes": 2147483648,
+    "gpu_number": 0
+  },
+  "version": "1.1.0"
+}
+JSON
+)
+
+body_file="$TMP_ROOT/register_body.json"
+status=$(curl -sS -o "$body_file" -w '%{http_code}' -H 'Content-Type: application/json' -X POST "$API_BASE/nodes" -d "$payload")
+body="$(cat "$body_file")"
+
+if [[ "$status" != "201" ]]; then
+  echo "[ERROR] Unexpected status code: $status" >&2
+  echo "$body" >&2
+  exit 1
+fi
+
+node_id=$(python3 - "$body_file" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    body = json.load(handle)
+print(body["id"])
+PY
+)
+
+echo "$body" > "$TMP_ROOT/last_response.json"
+echo "$node_id" > "$TMP_ROOT/node_id"
+
+list_file="$TMP_ROOT/nodes_list.json"
+curl -sS "$API_BASE/nodes" -o "$list_file"
+python3 - "$list_file" "$node_id" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    data = json.load(handle)
+node_id = sys.argv[2]
+assert any(item.get("id") == node_id for item in data), "node not in list"
+PY
+
+echo "[INFO] Registered node with id $node_id"
--- a/src/master/tests/scripts/04_reregister_and_error_cases.sh
+++ b/src/master/tests/scripts/04_reregister_and_error_cases.sh
@@ -0,0 +1,116 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+TMP_ROOT="$TEST_ROOT/tmp"
+API_BASE="http://localhost:31300/api/v1/master"
+NODE_ID="$(cat "$TMP_ROOT/node_id")"
+
+# 使用相同 ID 重注册，同时修改部分 meta/version 字段
+payload=$(cat <<JSON
+{
+  "id": "$NODE_ID",
+  "name": "dev-testuser-testinst-pod-0",
+  "type": "agent",
+  "meta_data": {
+    "hostname": "dev-testuser-testinst-pod-0",
+    "ip": "10.0.0.11",
+    "env": "dev",
+    "user": "testuser",
+    "instance": "testinst",
+    "cpu_number": 8,
+    "memory_in_bytes": 2147483648,
+    "gpu_number": 0
+  },
+  "version": "1.2.0"
+}
+JSON
+)
+
+status=$(curl -sS -o "$TMP_ROOT/reregister_response.json" -w '%{http_code}' -H 'Content-Type: application/json' -X POST "$API_BASE/nodes" -d "$payload")
+if [[ "$status" != "200" ]]; then
+  echo "[ERROR] 重注册返回非 200: $status" >&2
+  cat "$TMP_ROOT/reregister_response.json" >&2
+  exit 1
+fi
+
+python3 - "$TMP_ROOT/reregister_response.json" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+assert node["meta_data"]["ip"] == "10.0.0.11", node["meta_data"]
+assert node["meta_data"]["cpu_number"] == 8, node["meta_data"]
+assert node["version"] == "1.2.0", node
+PY
+
+echo "[INFO] 重注册成功，元数据已更新"
+
+# 未知 ID => 404
+unknown_payload=$(cat <<'JSON'
+{
+  "id": "A999",
+  "name": "dev-testuser-testinst-pod-0",
+  "type": "agent",
+  "meta_data": {
+    "hostname": "dev-testuser-testinst-pod-0",
+    "ip": "10.0.0.12",
+    "env": "dev",
+    "user": "testuser",
+    "instance": "testinst",
+    "cpu_number": 4,
+    "memory_in_bytes": 2147483648,
+    "gpu_number": 0
+  },
+  "version": "1.2.0"
+}
+JSON
+)
+
+status=$(curl -sS -o "$TMP_ROOT/unknown_id_response.json" -w '%{http_code}' -H 'Content-Type: application/json' -X POST "$API_BASE/nodes" -d "$unknown_payload")
+if [[ "$status" != "404" ]]; then
+  echo "[ERROR] 未知 ID 应返回 404，实际=$status" >&2
+  cat "$TMP_ROOT/unknown_id_response.json" >&2
+  exit 1
+fi
+
+echo "[INFO] 未知 ID 返回 404 验证通过"
+
+# id 与 name 不匹配 => 500，节点保持原名
+mismatch_payload=$(cat <<JSON
+{
+  "id": "$NODE_ID",
+  "name": "dev-testuser-testinst-pod-0-mismatch",
+  "type": "agent",
+  "meta_data": {
+    "hostname": "dev-testuser-testinst-pod-0-mismatch",
+    "ip": "10.0.0.13",
+    "env": "dev",
+    "user": "testuser",
+    "instance": "testinst",
+    "cpu_number": 4,
+    "memory_in_bytes": 2147483648,
+    "gpu_number": 0
+  },
+  "version": "1.2.0"
+}
+JSON
+)
+
+status=$(curl -sS -o "$TMP_ROOT/mismatch_response.json" -w '%{http_code}' -H 'Content-Type: application/json' -X POST "$API_BASE/nodes" -d "$mismatch_payload")
+if [[ "$status" != "500" ]]; then
+  echo "[ERROR] 名称不匹配应返回 500，实际=$status" >&2
+  cat "$TMP_ROOT/mismatch_response.json" >&2
+  exit 1
+fi
+
+# 验证名称仍保持正确
+curl -sS "$API_BASE/nodes/$NODE_ID" -o "$TMP_ROOT/post_mismatch_detail.json"
+python3 - "$TMP_ROOT/post_mismatch_detail.json" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+assert node["name"] == "dev-testuser-testinst-pod-0", node["name"]
+PY
+
+echo "[INFO] 名称不匹配返回 500，且原始节点未被篡改"
--- a/src/master/tests/scripts/05_status_report_via_curl.sh
+++ b/src/master/tests/scripts/05_status_report_via_curl.sh
@@ -0,0 +1,98 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+TMP_ROOT="$TEST_ROOT/tmp"
+API_BASE="http://localhost:31300/api/v1/master"
+
+node_id="$(cat "$TMP_ROOT/node_id")"
+
+payload=$(python3 - <<'PY'
+import json
+from datetime import datetime, timezone
+body = {
+    "timestamp": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
+    "health": {
+        "log-fluentbit": {"status": "healthy", "timestamp": "2023-10-05T12:05:00Z"},
+        "metric-node-exporter": {"status": "healthy", "timestamp": "2023-10-05T12:05:00Z"}
+    }
+}
+print(json.dumps(body))
+PY
+)
+
+response=$(curl -sS -w '\n%{http_code}' -H 'Content-Type: application/json' -X PUT "$API_BASE/nodes/$node_id/status" -d "$payload")
+body="$(echo "$response" | head -n -1)"
+status="$(echo "$response" | tail -n1)"
+
+if [[ "$status" != "200" ]]; then
+  echo "[ERROR] Status update failed with code $status" >&2
+  echo "$body" >&2
+  exit 1
+fi
+
+echo "$body" > "$TMP_ROOT/last_response.json"
+
+sleep 3
+
+detail_file="$TMP_ROOT/status_detail.json"
+curl -sS "$API_BASE/nodes/$node_id" -o "$detail_file"
+python3 - "$detail_file" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+assert node["status"] == "online", f"Expected online, got {node['status']}"
+assert "log-fluentbit" in node["health"], node["health"].keys()
+PY
+
+echo "[INFO] Status report successful and node is online"
+
+# 等待超过 offline 阈值，验证会自动转为 offline
+sleep 7
+
+offline_detail_file="$TMP_ROOT/status_offline.json"
+curl -sS "$API_BASE/nodes/$node_id" -o "$offline_detail_file"
+python3 - "$offline_detail_file" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+assert node["status"] == "offline", f"Expected offline, got {node['status']}"
+PY
+
+echo "[INFO] Node transitioned to offline as expected"
+
+# 再次上报健康，触发状态回到 online
+payload=$(python3 - <<'PY'
+import json
+from datetime import datetime, timezone
+body = {
+    "timestamp": datetime.now(timezone.utc).replace(microsecond=0).isoformat().replace("+00:00", "Z"),
+    "health": {
+        "log-fluentbit": {"status": "healthy", "timestamp": "2023-10-05T12:05:00Z"},
+        "metric-node-exporter": {"status": "healthy", "timestamp": "2023-10-05T12:05:00Z"}
+    }
+}
+print(json.dumps(body))
+PY
+)
+
+curl -sS -o "$TMP_ROOT/second_status_response.json" -w '%{http_code}' -H 'Content-Type: application/json' -X PUT "$API_BASE/nodes/$node_id/status" -d "$payload" > "$TMP_ROOT/second_status_code"
+if [[ $(cat "$TMP_ROOT/second_status_code") != "200" ]]; then
+  echo "[ERROR] Second status update failed" >&2
+  cat "$TMP_ROOT/second_status_response.json" >&2
+  exit 1
+fi
+
+sleep 3
+
+final_detail_file="$TMP_ROOT/status_back_online.json"
+curl -sS "$API_BASE/nodes/$node_id" -o "$final_detail_file"
+python3 - "$final_detail_file" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+assert node["status"] == "online", f"Expected online after second report, got {node['status']}"
+PY
+
+echo "[INFO] Node transitioned back to online after new status report"
--- a/src/master/tests/scripts/06_config_update_and_nodes_json.sh
+++ b/src/master/tests/scripts/06_config_update_and_nodes_json.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+TMP_ROOT="$TEST_ROOT/tmp"
+PRIVATE_ROOT="$TEST_ROOT/private"
+API_BASE="http://localhost:31300/api/v1/master"
+NODE_ID="$(cat "$TMP_ROOT/node_id")"
+
+payload='{"config":{"log_level":"debug"},"label":["gpu","exp001"]}'
+
+response=$(curl -sS -w '\n%{http_code}' -H 'Content-Type: application/json' -X PUT "$API_BASE/nodes/$NODE_ID/config" -d "$payload")
+body="$(echo "$response" | head -n -1)"
+status="$(echo "$response" | tail -n1)"
+
+if [[ "$status" != "200" ]]; then
+  echo "[ERROR] Config update failed: $status" >&2
+  echo "$body" >&2
+  exit 1
+fi
+
+sleep 2
+
+nodes_json_path="$PRIVATE_ROOT/argus/metric/prometheus/nodes.json"
+if [[ ! -f "$nodes_json_path" ]]; then
+  echo "[ERROR] nodes.json not generated at $nodes_json_path" >&2
+  exit 1
+fi
+
+# 确保节点处于 online 状态，避免因等待导致 nodes.json 为空
+curl -sS "$API_BASE/nodes/$NODE_ID" -o "$TMP_ROOT/config_detail.json"
+if ! python3 - "$TMP_ROOT/config_detail.json" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+if node["status"] != "online":
+    raise SystemExit(1)
+PY
+then
+  payload='{"timestamp":"2025-09-24T00:00:00Z","health":{"log-fluentbit":{"status":"healthy"}}}'
+  curl -sS -o "$TMP_ROOT/config_second_report.json" -w '%{http_code}' -H 'Content-Type: application/json' -X PUT "$API_BASE/nodes/$NODE_ID/status" -d "$payload" > "$TMP_ROOT/config_second_code"
+  sleep 2
+fi
+
+python3 - "$nodes_json_path" <<'PY'
+import json, sys
+from pathlib import Path
+path = Path(sys.argv[1])
+content = json.loads(path.read_text())
+assert isinstance(content, list) and len(content) == 1
+entry = content[0]
+assert entry["labels"] == ["gpu", "exp001"], entry
+PY
+
+echo "[INFO] Config updated and nodes.json verified"
--- a/src/master/tests/scripts/07_stats_single_node.sh
+++ b/src/master/tests/scripts/07_stats_single_node.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+PRIVATE_ROOT="$TEST_ROOT/private"
+TMP_ROOT="$TEST_ROOT/tmp"
+API_BASE="http://localhost:31300/api/v1/master"
+NODE_ID="$(cat "$TMP_ROOT/node_id")"
+
+sleep 7
+
+detail_file="$TMP_ROOT/offline_detail.json"
+curl -sS "$API_BASE/nodes/$NODE_ID" -o "$detail_file"
+python3 - "$detail_file" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+assert node["status"] == "offline", f"Expected offline, got {node['status']}"
+PY
+
+stats_file="$TMP_ROOT/stats.json"
+curl -sS "$API_BASE/nodes/statistics" -o "$stats_file"
+python3 - "$stats_file" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    stats = json.load(handle)
+assert stats["total"] == 1
+found = {item["status"]: item["count"] for item in stats["status_statistics"]}
+assert found.get("offline") == 1
+PY
+
+nodes_json_path="$PRIVATE_ROOT/argus/metric/prometheus/nodes.json"
+python3 - "$nodes_json_path" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    content = json.load(handle)
+assert content == [], content
+PY
+
+echo "[INFO] Offline transition and statistics validated"
--- a/src/master/tests/scripts/08_multi_node_stats.sh
+++ b/src/master/tests/scripts/08_multi_node_stats.sh
@@ -0,0 +1,106 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+PRIVATE_ROOT="$TEST_ROOT/private"
+TMP_ROOT="$TEST_ROOT/tmp"
+API_BASE="http://localhost:31300/api/v1/master"
+
+# 注册第二个节点 A2（保持在线）
+second_payload=$(cat <<'JSON'
+{
+  "name": "dev-testuser-testinst-pod-1",
+  "type": "agent",
+  "meta_data": {
+    "hostname": "dev-testuser-testinst-pod-1",
+    "ip": "10.0.0.11",
+    "env": "dev",
+    "user": "testuser",
+    "instance": "testinst",
+    "cpu_number": 8,
+    "memory_in_bytes": 2147483648,
+    "gpu_number": 0
+  },
+  "version": "1.1.0"
+}
+JSON
+)
+
+status=$(curl -sS -o "$TMP_ROOT/second_register.json" -w '%{http_code}' -H 'Content-Type: application/json' -X POST "$API_BASE/nodes" -d "$second_payload")
+if [[ "$status" != "201" ]]; then
+  echo "[ERROR] Second node registration failed: $status" >&2
+  cat "$TMP_ROOT/second_register.json" >&2
+  exit 1
+fi
+SECOND_NODE_ID=$(python3 - "$TMP_ROOT/second_register.json" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    data = json.load(handle)
+print(data["id"])
+PY
+)
+
+echo "$SECOND_NODE_ID" > "$TMP_ROOT/second_node_id"
+
+echo "[INFO] Second node registered with id $SECOND_NODE_ID"
+
+# A2 上报健康信息，保持 online
+status_payload='{"timestamp":"2025-09-24T00:00:00Z","health":{"log-fluentbit":{"status":"healthy"}}}'
+status=$(curl -sS -o "$TMP_ROOT/second_status.json" -w '%{http_code}' -H 'Content-Type: application/json' -X PUT "$API_BASE/nodes/$SECOND_NODE_ID/status" -d "$status_payload")
+if [[ "$status" != "200" ]]; then
+  echo "[ERROR] Second node status update failed: $status" >&2
+  cat "$TMP_ROOT/second_status.json" >&2
+  exit 1
+fi
+
+# 等待调度器把第二节点标记为 online
+second_online=false
+for _ in {1..10}; do
+  sleep 1
+  curl -sS "$API_BASE/nodes/$SECOND_NODE_ID" -o "$TMP_ROOT/second_detail.json" || continue
+  if python3 - "$TMP_ROOT/second_detail.json" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+if node["status"] != "online":
+    raise SystemExit(1)
+PY
+  then
+    second_online=true
+    break
+  fi
+done
+
+if [[ "$second_online" != true ]]; then
+  echo "[ERROR] Second node did not become online" >&2
+  exit 1
+fi
+
+# 再次获取统计信息
+stats_file="$TMP_ROOT/multi_stats.json"
+curl -sS "$API_BASE/nodes/statistics" -o "$stats_file"
+python3 - "$stats_file" "$TMP_ROOT/node_id" "$TMP_ROOT/second_node_id" <<'PY'
+import json, sys, pathlib
+with open(sys.argv[1]) as handle:
+    stats = json.load(handle)
+first_id = pathlib.Path(sys.argv[2]).read_text().strip()
+second_id = pathlib.Path(sys.argv[3]).read_text().strip()
+assert stats["total"] == 2, stats
+found = {item["status"]: item["count"] for item in stats["status_statistics"]}
+assert found.get("offline") == 1, found
+assert found.get("online") == 1, found
+PY
+
+# 验证 nodes.json 只包含在线节点（应只有第二个 A2）
+nodes_json_path="$PRIVATE_ROOT/argus/metric/prometheus/nodes.json"
+python3 - "$nodes_json_path" "$SECOND_NODE_ID" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    content = json.load(handle)
+expected_id = sys.argv[2]
+assert len(content) == 1, content
+assert content[0]["node_id"] == expected_id, content
+PY
+
+echo "[INFO] Multi-node statistics and nodes.json validated"
--- a/src/master/tests/scripts/09_restart_persistence.sh
+++ b/src/master/tests/scripts/09_restart_persistence.sh
@@ -0,0 +1,161 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+PRIVATE_ROOT="$TEST_ROOT/private"
+TMP_ROOT="$TEST_ROOT/tmp"
+API_BASE="http://localhost:31300/api/v1/master"
+ROOT_BASE="http://localhost:31300"
+DB_PATH="$PRIVATE_ROOT/argus/master/db.sqlite3"
+
+compose() {
+  if docker compose version >/dev/null 2>&1; then
+    docker compose "$@"
+  else
+    docker-compose "$@"
+  fi
+}
+
+if [[ ! -f "$TMP_ROOT/node_id" ]]; then
+  echo "[ERROR] 主节点 ID 缺失，请先执行前置用例" >&2
+  exit 1
+fi
+
+if [[ ! -f "$TMP_ROOT/second_node_id" ]]; then
+  echo "[ERROR] 第二个节点 ID 缺失，请先执行多节点场景脚本" >&2
+  exit 1
+fi
+
+if [[ ! -f "$DB_PATH" ]]; then
+  echo "[ERROR] 持久化数据库缺失: $DB_PATH" >&2
+  exit 1
+fi
+
+NODE_ID="$(cat "$TMP_ROOT/node_id")"
+SECOND_NODE_ID="$(cat "$TMP_ROOT/second_node_id")"
+
+# 在重启前抓取节点详情与节点文件、统计信息，作为对比基线
+first_before="$TMP_ROOT/${NODE_ID}_pre_restart.json"
+second_before="$TMP_ROOT/${SECOND_NODE_ID}_pre_restart.json"
+curl -sS "$API_BASE/nodes/$NODE_ID" -o "$first_before"
+curl -sS "$API_BASE/nodes/$SECOND_NODE_ID" -o "$second_before"
+
+nodes_json_before="$TMP_ROOT/nodes_json_pre_restart.json"
+cp "$PRIVATE_ROOT/argus/metric/prometheus/nodes.json" "$nodes_json_before"
+
+stats_before="$TMP_ROOT/stats_pre_restart.json"
+curl -sS "$API_BASE/nodes/statistics" -o "$stats_before"
+
+# 重启 master 容器，模拟服务重启后的持久化场景
+pushd "$TEST_ROOT" >/dev/null
+compose restart master
+popd >/dev/null
+
+# 等待 /readyz 恢复 200
+for _ in {1..30}; do
+  status=$(curl -s -o /dev/null -w '%{http_code}' "$ROOT_BASE/readyz" || true)
+  if [[ "$status" == "200" ]]; then
+    break
+  fi
+  sleep 1
+done
+
+if [[ "${status:-}" != "200" ]]; then
+  echo "[ERROR] master 容器重启后未恢复健康状态，readyz=$status" >&2
+  exit 1
+fi
+
+sleep 2
+
+first_after="$TMP_ROOT/${NODE_ID}_post_restart.json"
+second_after="$TMP_ROOT/${SECOND_NODE_ID}_post_restart.json"
+curl -sS "$API_BASE/nodes/$NODE_ID" -o "$first_after"
+curl -sS "$API_BASE/nodes/$SECOND_NODE_ID" -o "$second_after"
+
+# 对比重启前后的节点关键信息，确保无丢失
+python3 - "$first_before" "$first_after" <<'PY'
+import json, sys
+before_path, after_path = sys.argv[1:3]
+with open(before_path, 'r', encoding='utf-8') as handle:
+    before = json.load(handle)
+with open(after_path, 'r', encoding='utf-8') as handle:
+    after = json.load(handle)
+keys = [
+    "id",
+    "name",
+    "type",
+    "version",
+    "register_time",
+    "meta_data",
+    "config",
+    "label",
+    "health",
+    "last_report",
+    "agent_last_report",
+    "status",
+]
+for key in keys:
+    if before.get(key) != after.get(key):
+        raise AssertionError(f"Key {key} changed after restart: {before.get(key)} -> {after.get(key)}")
+PY
+
+python3 - "$second_before" "$second_after" <<'PY'
+import json, sys
+before_path, after_path = sys.argv[1:3]
+with open(before_path, 'r', encoding='utf-8') as handle:
+    before = json.load(handle)
+with open(after_path, 'r', encoding='utf-8') as handle:
+    after = json.load(handle)
+keys = [
+    "id",
+    "name",
+    "type",
+    "version",
+    "register_time",
+    "meta_data",
+    "config",
+    "label",
+    "health",
+    "last_report",
+    "agent_last_report",
+    "status",
+]
+for key in keys:
+    if before.get(key) != after.get(key):
+        raise AssertionError(f"Key {key} changed after restart: {before.get(key)} -> {after.get(key)}")
+PY
+
+# 对比重启前后的 nodes.json 与统计信息，验证持久化一致性
+nodes_json_after="$TMP_ROOT/nodes_json_post_restart.json"
+cp "$PRIVATE_ROOT/argus/metric/prometheus/nodes.json" "$nodes_json_after"
+
+stats_after="$TMP_ROOT/stats_after_restart.json"
+curl -sS "$API_BASE/nodes/statistics" -o "$stats_after"
+
+python3 - "$nodes_json_before" "$nodes_json_after" <<'PY'
+import json, sys
+with open(sys.argv[1], 'r', encoding='utf-8') as handle:
+    before = json.load(handle)
+with open(sys.argv[2], 'r', encoding='utf-8') as handle:
+    after = json.load(handle)
+if before != after:
+    raise AssertionError(f"nodes.json changed after restart: {before} -> {after}")
+PY
+
+python3 - "$stats_before" "$stats_after" <<'PY'
+import json, sys
+with open(sys.argv[1], 'r', encoding='utf-8') as handle:
+    before = json.load(handle)
+with open(sys.argv[2], 'r', encoding='utf-8') as handle:
+    after = json.load(handle)
+if before != after:
+    raise AssertionError(f"Statistics changed after restart: {before} -> {after}")
+PY
+
+if [[ ! -s "$DB_PATH" ]]; then
+  echo "[ERROR] 数据库文件为空，疑似未持久化" >&2
+  exit 1
+fi
+
+echo "[INFO] Master 重启后持久化数据校验通过"
--- a/src/master/tests/scripts/10_down.sh
+++ b/src/master/tests/scripts/10_down.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+PRIVATE_ROOT="$TEST_ROOT/private"
+TMP_ROOT="$TEST_ROOT/tmp"
+
+compose() {
+  if docker compose version >/dev/null 2>&1; then
+    docker compose "$@"
+  else
+    docker-compose "$@"
+  fi
+}
+
+pushd "$TEST_ROOT" >/dev/null
+compose down --remove-orphans
+popd >/dev/null
+
+rm -rf "$TMP_ROOT"
+rm -rf "$PRIVATE_ROOT"
+
+echo "[INFO] Master E2E environment cleaned up"