[#1] agent模块开发

2025-09-24 04:19:37 +00:00
parent 5e5342ce20
commit bc016826f1
25 changed files with 1123 additions and 0 deletions
--- a/src/agent/tests/scripts/00_e2e_test.sh
+++ b/src/agent/tests/scripts/00_e2e_test.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+SCRIPTS=(
+  "01_bootstrap.sh"
+  "02_up.sh"
+  "03_wait_and_assert_registration.sh"
+  "04_write_health_files.sh"
+  "05_assert_status_on_master.sh"
+  "06_restart_agent_and_reregister.sh"
+  "07_down.sh"
+)
+
+for script in "${SCRIPTS[@]}"; do
+  echo "[TEST] Running $script"
+  "$SCRIPT_DIR/$script"
+  echo "[TEST] $script completed"
+  echo
+done
+
+echo "[TEST] Agent module E2E tests completed"
--- a/src/agent/tests/scripts/01_bootstrap.sh
+++ b/src/agent/tests/scripts/01_bootstrap.sh
@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+AGENT_ROOT="$(cd "$TEST_ROOT/.." && pwd)"
+MASTER_ROOT="$(cd "$AGENT_ROOT/../master" && pwd)"
+PRIVATE_ROOT="$TEST_ROOT/private"
+TMP_ROOT="$TEST_ROOT/tmp"
+
+AGENT_HOSTNAME="dev-e2euser-e2einst-pod-0"
+AGENT_CONFIG_DIR="$PRIVATE_ROOT/argus/agent/$AGENT_HOSTNAME"
+AGENT_HEALTH_DIR="$PRIVATE_ROOT/argus/agent/health/$AGENT_HOSTNAME"
+MASTER_PRIVATE_DIR="$PRIVATE_ROOT/argus/master"
+METRIC_PRIVATE_DIR="$PRIVATE_ROOT/argus/metric/prometheus"
+
+mkdir -p "$AGENT_CONFIG_DIR"
+mkdir -p "$AGENT_HEALTH_DIR"
+mkdir -p "$MASTER_PRIVATE_DIR"
+mkdir -p "$METRIC_PRIVATE_DIR"
+mkdir -p "$TMP_ROOT"
+
+cat > "$AGENT_CONFIG_DIR/config" <<CONFIG
+HOSTNAME=$AGENT_HOSTNAME
+NODE_FILE=/private/argus/agent/$AGENT_HOSTNAME/node.json
+VERSION=1.1.0
+MASTER_ENDPOINT=http://master:3000
+REPORT_INTERVAL_SECONDS=2
+SUBMODULE_HEALTH_FILE_DIR=/private/argus/agent/health/{hostname}/
+GPU_NUMBER=
+IP_OVERRIDE=
+CONFIG
+
+touch "$AGENT_HEALTH_DIR/.keep"
+
+pushd "$MASTER_ROOT" >/dev/null
+./scripts/build_images.sh --tag argus-master:dev
+popd >/dev/null
+
+pushd "$AGENT_ROOT" >/dev/null
+./scripts/build_images.sh --tag argus-agent:dev
+popd >/dev/null
+
+echo "[INFO] Agent E2E bootstrap complete"
--- a/src/agent/tests/scripts/02_up.sh
+++ b/src/agent/tests/scripts/02_up.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+compose() {
+  if docker compose version >/dev/null 2>&1; then
+    docker compose "$@"
+  else
+    docker-compose "$@"
+  fi
+}
+
+docker container rm -f argus-agent-e2e >/dev/null 2>&1 || true
+
+docker network rm tests_default >/dev/null 2>&1 || true
+
+pushd "$TEST_ROOT" >/dev/null
+compose down --remove-orphans || true
+compose up -d
+popd >/dev/null
+
+echo "[INFO] Master+Agent stack started"
--- a/src/agent/tests/scripts/03_wait_and_assert_registration.sh
+++ b/src/agent/tests/scripts/03_wait_and_assert_registration.sh
@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+TMP_ROOT="$TEST_ROOT/tmp"
+API_BASE="http://localhost:32300/api/v1/master"
+AGENT_HOSTNAME="dev-e2euser-e2einst-pod-0"
+NODE_FILE="$TEST_ROOT/private/argus/agent/$AGENT_HOSTNAME/node.json"
+
+mkdir -p "$TMP_ROOT"
+
+node_id=""
+for _ in {1..30}; do
+  sleep 2
+  response=$(curl -sS "$API_BASE/nodes" || true)
+  if [[ -z "$response" ]]; then
+    continue
+  fi
+  list_file="$TMP_ROOT/nodes_list.json"
+  echo "$response" > "$list_file"
+  node_id=$(python3 - "$list_file" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    nodes = json.load(handle)
+print(nodes[0]["id"] if nodes else "")
+PY
+)
+  if [[ -n "$node_id" ]]; then
+    break
+  fi
+ done
+
+if [[ -z "$node_id" ]]; then
+  echo "[ERROR] Agent did not register within timeout" >&2
+  exit 1
+fi
+
+echo "$node_id" > "$TMP_ROOT/node_id"
+
+if [[ ! -f "$NODE_FILE" ]]; then
+  echo "[ERROR] node.json not created at $NODE_FILE" >&2
+  exit 1
+fi
+
+python3 - "$NODE_FILE" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+assert "id" in node and node["id"], "node.json missing id"
+PY
+
+detail_file="$TMP_ROOT/initial_detail.json"
+curl -sS "$API_BASE/nodes/$node_id" -o "$detail_file"
+python3 - "$detail_file" "$TMP_ROOT/initial_ip" <<'PY'
+import json, sys, pathlib
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+ip = node["meta_data"].get("ip")
+if not ip:
+    raise SystemExit("meta_data.ip missing")
+pathlib.Path(sys.argv[2]).write_text(ip)
+PY
+
+echo "[INFO] Agent registered with node id $node_id"
--- a/src/agent/tests/scripts/04_write_health_files.sh
+++ b/src/agent/tests/scripts/04_write_health_files.sh
@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+HEALTH_DIR="$TEST_ROOT/private/argus/agent/health/dev-e2euser-e2einst-pod-0"
+
+cat > "$HEALTH_DIR/log-fluentbit.json" <<JSON
+{
+  "status": "healthy",
+  "timestamp": "2023-10-05T12:05:00Z"
+}
+JSON
+
+cat > "$HEALTH_DIR/metric-node-exporter.json" <<JSON
+{
+  "status": "healthy",
+  "timestamp": "2023-10-05T12:05:00Z"
+}
+JSON
+
+echo "[INFO] Health files written"
--- a/src/agent/tests/scripts/05_assert_status_on_master.sh
+++ b/src/agent/tests/scripts/05_assert_status_on_master.sh
@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+TMP_ROOT="$TEST_ROOT/tmp"
+API_BASE="http://localhost:32300/api/v1/master"
+NODE_ID="$(cat "$TMP_ROOT/node_id")"
+NODES_JSON="$TEST_ROOT/private/argus/metric/prometheus/nodes.json"
+
+success=false
+detail_file="$TMP_ROOT/agent_status_detail.json"
+for _ in {1..20}; do
+  sleep 2
+  if ! curl -sS "$API_BASE/nodes/$NODE_ID" -o "$detail_file"; then
+    continue
+  fi
+  if python3 - "$detail_file" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+if node["status"] != "online":
+    raise SystemExit(1)
+health = node.get("health", {})
+if "log-fluentbit" not in health or "metric-node-exporter" not in health:
+    raise SystemExit(1)
+PY
+  then
+    success=true
+    break
+  fi
+done
+
+if [[ "$success" != true ]]; then
+  echo "[ERROR] Node did not report health data in time" >&2
+  exit 1
+fi
+
+if [[ ! -f "$NODES_JSON" ]]; then
+  echo "[ERROR] nodes.json missing at $NODES_JSON" >&2
+  exit 1
+fi
+
+python3 - "$NODES_JSON" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    nodes = json.load(handle)
+assert len(nodes) == 1, nodes
+entry = nodes[0]
+assert entry["node_id"], entry
+PY
+
+echo "[INFO] Master reflects agent health and nodes.json entries"
--- a/src/agent/tests/scripts/06_restart_agent_and_reregister.sh
+++ b/src/agent/tests/scripts/06_restart_agent_and_reregister.sh
@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+TMP_ROOT="$TEST_ROOT/tmp"
+API_BASE="http://localhost:32300/api/v1/master"
+NODE_ID="$(cat "$TMP_ROOT/node_id")"
+AGENT_HOSTNAME="dev-e2euser-e2einst-pod-0"
+NETWORK_NAME="tests_default"
+NEW_AGENT_IP="172.28.0.200"
+
+compose() {
+  if docker compose version >/dev/null 2>&1; then
+    docker compose "$@"
+  else
+    docker-compose "$@"
+  fi
+}
+
+before_file="$TMP_ROOT/before_restart.json"
+curl -sS "$API_BASE/nodes/$NODE_ID" -o "$before_file"
+prev_last_updated=$(python3 - "$before_file" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+print(node.get("last_updated", ""))
+PY
+)
+prev_ip=$(python3 - "$before_file" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+print(node["meta_data"].get("ip", ""))
+PY
+)
+initial_ip=$(cat "$TMP_ROOT/initial_ip")
+if [[ "$prev_ip" != "$initial_ip" ]]; then
+  echo "[ERROR] Expected initial IP $initial_ip, got $prev_ip" >&2
+  exit 1
+fi
+
+pushd "$TEST_ROOT" >/dev/null
+compose rm -sf agent
+popd >/dev/null
+
+docker container rm -f argus-agent-e2e >/dev/null 2>&1 || true
+
+AGENT_DIR="$TEST_ROOT/private/argus/agent/$AGENT_HOSTNAME"
+HEALTH_DIR="$TEST_ROOT/private/argus/agent/health/$AGENT_HOSTNAME"
+
+# 先以 sleep 方式启动容器，确保我们掌握注册时的网络状态
+if ! docker run -d \
+  --name argus-agent-e2e \
+  --hostname "$AGENT_HOSTNAME" \
+  --network "$NETWORK_NAME" \
+  --ip "$NEW_AGENT_IP" \
+  -v "$AGENT_DIR:/private/argus/agent/$AGENT_HOSTNAME" \
+  -v "$HEALTH_DIR:/private/argus/agent/health/$AGENT_HOSTNAME" \
+  argus-agent:dev \
+  sleep 300 >/dev/null; then
+  echo "[ERROR] Failed to start agent container with custom IP" >&2
+  exit 1
+fi
+
+# 在容器内启动真实 agent 进程
+if ! docker exec -d argus-agent-e2e python -m app.main; then
+  echo "[ERROR] Failed to spawn agent process inside container" >&2
+  exit 1
+fi
+
+success=false
+detail_file="$TMP_ROOT/post_restart.json"
+for _ in {1..20}; do
+  sleep 3
+  if ! curl -sS "$API_BASE/nodes/$NODE_ID" -o "$detail_file"; then
+    continue
+  fi
+  if python3 - "$detail_file" "$prev_last_updated" "$NODE_ID" "$prev_ip" "$NEW_AGENT_IP" <<'PY'
+import json, sys
+with open(sys.argv[1]) as handle:
+    node = json.load(handle)
+prev_last_updated = sys.argv[2]
+expected_id = sys.argv[3]
+old_ip = sys.argv[4]
+expected_ip = sys.argv[5]
+last_updated = node.get("last_updated")
+current_ip = node["meta_data"].get("ip")
+assert node["id"] == expected_id
+if current_ip != expected_ip:
+    raise SystemExit(1)
+if current_ip == old_ip:
+    raise SystemExit(1)
+if not last_updated or last_updated == prev_last_updated:
+    raise SystemExit(1)
+PY
+  then
+    success=true
+    break
+  fi
+done
+
+if [[ "$success" != true ]]; then
+  echo "[ERROR] Agent did not report expected new IP $NEW_AGENT_IP after restart" >&2
+  exit 1
+fi
+
+echo "[INFO] Agent restart produced successful re-registration with IP change"
--- a/src/agent/tests/scripts/07_down.sh
+++ b/src/agent/tests/scripts/07_down.sh
@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+compose() {
+  if docker compose version >/dev/null 2>&1; then
+    docker compose "$@"
+  else
+    docker-compose "$@"
+  fi
+}
+
+docker container rm -f argus-agent-e2e >/dev/null 2>&1 || true
+
+pushd "$TEST_ROOT" >/dev/null
+compose down --remove-orphans
+popd >/dev/null
+
+rm -rf "$TEST_ROOT/private"
+rm -rf "$TEST_ROOT/tmp"
+
+echo "[INFO] Agent E2E environment cleaned up"