dev_1.0.0_xuxt_3 完成web和alert的集成测试 (#38)

Co-authored-by: xiuting.xu <xiutingxt.xu@gmail.com>
Reviewed-on: #38
Reviewed-by: huhy <husteryezi@163.com>
Reviewed-by: yuyr <yuyr@zgclab.edu.cn>
Reviewed-by: sundapeng <sundp@mail.zgclab.edu.cn>

src/sys/tests/README.md

@@ -47,6 +47,8 @@
   - `./scripts/11_metric_node_install.sh` 在 CPU 节点安装并验证端点
   - `./scripts/12_metric_gpu_install.sh` 在 GPU 节点安装并等待 9100/9400 就绪（仅启用 GPU 时）
   - `./scripts/13_metric_verify.sh` 对 master/Prometheus/数据面/Grafana 做综合校验（含 GPU 时校验 dcgm 指标）
+  - `./scripts/15_alert_verify.sh` 对alertmanager进行校验
+  - `./scripts/16_web_verify.sh` 对web页面进行校验综合校验。
   - `./scripts/14_metric_cleanup.sh` 清理 FTP 产物
   - `./scripts/09_down.sh` 回收容器、网络并清理 `private*/`、`tmp/`
 
@@ -133,6 +135,13 @@
 - `09_down.sh`
   - 目的：栈销毁与环境清理；必要时使用临时容器修正属主再删除 `private*` 目录
 
+- `15_alert_verify.sh`
+  - 目的：验证alertmanager的可用性、Prometheus到alertmanager的连通性。
+  - 操作：在Prometheus中增加一个恒为真的告警规则，查看alertmanager是否收到该告警
+- `16_web_verify.sh`
+  - 目的：验证web页面是否可用。
+  - 使用playwright分别验证各个模块的页面是否可用，以及符合预期。
+
 ---
 
 ### 常见问题与排查

src/sys/tests/scripts/00_e2e_test.sh

@@ -55,6 +55,8 @@ SCRIPTS=(
   "11_metric_node_install.sh"
   "12_metric_gpu_install.sh"
   "13_metric_verify.sh"
+  "15_alert_verify.sh"
+  "16_web_verify.sh"
 )
 
 # 如未禁用清理，则追加清理与下线步骤（保持原有顺序）

src/sys/tests/scripts/15_alert_verify.sh

@@ -0,0 +1,103 @@
+#!/bin/bash
+# verify_alertmanager.sh
+# Verify the communication between Prometheus and Alertmanager after deployment
+
+set -euo pipefail
+
+echo "[INFO] Verifying Prometheus ↔ Alertmanager communication..."
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+TMP_DIR="$TEST_ROOT/tmp"
+mkdir -p "$TMP_DIR"
+
+PRIVATE_CORE="$TEST_ROOT/private"
+
+#=============================
+# Load environment variables
+#=============================
+if [[ -f "$TEST_ROOT/.env" ]]; then
+  set -a; source "$TEST_ROOT/.env"; set +a
+fi
+
+#=============================
+# Basic configuration
+#=============================
+PROM_URL="http://localhost:${PROMETHEUS_PORT:-9090}"
+ALERT_URL="http://localhost:${ALERTMANAGER_PORT:-9093}"
+RULE_DIR="$PRIVATE_CORE/argus/metric/prometheus/rules"
+TMP_RULE="$TMP_DIR/test_rule.yml"
+
+#=============================
+# Helper functions
+#=============================
+GREEN="\033[32m"; RED="\033[31m"; YELLOW="\033[33m"; RESET="\033[0m"
+
+log_info()    { echo -e "${YELLOW}[INFO]${RESET} $1"; }
+log_success() { echo -e "${GREEN}[OK]${RESET} $1"; }
+log_warn()    { echo -e "${YELLOW}[WARN]${RESET} $1"; }
+log_error()   { echo -e "${RED}[ERROR]${RESET} $1"; }
+
+fail_exit() { log_error "$1"; exit 1; }
+
+#=============================
+# Step 1: Check Alertmanager accessibility
+#=============================
+log_info "Checking Alertmanager status..."
+if curl -sSf "${ALERT_URL}/api/v2/status" >/dev/null 2>&1; then
+  log_success "Alertmanager is reachable at ${ALERT_URL}"
+else
+  fail_exit "Alertmanager is not reachable. Please check container or port mapping."
+fi
+
+#=============================
+# Step 2: Create and load a temporary test alert rule
+#=============================
+log_info "Creating temporary alert rule at ${TMP_RULE}..."
+cat <<EOF > "${TMP_RULE}"
+groups:
+- name: deploy-verify-group
+  rules:
+  - alert: DeployVerifyAlert
+    expr: vector(1)
+    labels:
+      severity: warning
+    annotations:
+      summary: "Deployment verification alert"
+EOF
+
+mkdir -p "${RULE_DIR}"
+cp "${TMP_RULE}" "${RULE_DIR}/test_rule.yml"
+
+log_info "Reloading Prometheus to apply the test rule..."
+if curl -s -X POST "${PROM_URL}/-/reload" >/dev/null; then
+  log_success "Prometheus successfully reloaded rules"
+else
+  fail_exit "Failed to reload Prometheus. Check API accessibility."
+fi
+
+#=============================
+# Step 3: Verify alert received by Alertmanager
+#=============================
+log_info "Waiting for alert propagation (~30 seconds)..."
+sleep 30
+
+if curl -s "${ALERT_URL}/api/v2/alerts" | grep -q "DeployVerifyAlert"; then
+  log_success "Prometheus → Alertmanager alert path verified successfully"
+else
+  fail_exit "DeployVerifyAlert not found in Alertmanager. Check configuration or network."
+fi
+
+#=============================
+# Step 4: Cleanup test rule
+#=============================
+log_info "Cleaning up temporary alert rule..."
+rm -f "${RULE_DIR}/test_rule.yml" "${TMP_RULE}"
+
+if curl -s -X POST "${PROM_URL}/-/reload" >/dev/null; then
+  log_success "Prometheus successfully reloaded after cleanup"
+else
+  log_warn "Prometheus reload after cleanup failed. Please check manually."
+fi
+
+log_success "Alertmanager verification completed successfully. Communication with Prometheus is healthy."

src/sys/tests/scripts/16_web_verify.sh

@@ -0,0 +1,115 @@
+#!/usr/bin/env bash
+# verify-web-test.sh
+# Verify frontend service availability and run Playwright end-to-end tests
+
+set -euo pipefail
+
+echo '[INFO] Verifying Web frontend...'
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+TEST_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
+REPO_ROOT="$(cd "$TEST_ROOT/../../.." && pwd)"
+WEB_DIR="$REPO_ROOT/src/web"
+
+#=============================
+# Load environment variables
+#=============================
+if [[ -f "$TEST_ROOT/.env" ]]; then
+  set -a; source "$TEST_ROOT/.env"; set +a
+fi
+
+REPORT_DIR="$WEB_DIR/playwright-report"
+FRONTEND_URL="http://localhost:${WEB_PROXY_PORT_8080:-8080}"
+TIMEOUT=120  # max wait time (seconds) for frontend to be ready
+
+#=============================
+# Helper functions
+#=============================
+GREEN="\033[32m"; RED="\033[31m"; YELLOW="\033[33m"; RESET="\033[0m"
+
+log_info()    { echo -e "${YELLOW}[INFO]${RESET} $1"; }
+log_success() { echo -e "${GREEN}[OK]${RESET} $1"; }
+log_warn()    { echo -e "${YELLOW}[WARN]${RESET} $1"; }
+log_error()   { echo -e "${RED}[ERROR]${RESET} $1"; }
+
+fail_exit() { log_error "$1"; exit 1; }
+
+#=============================
+# Step 1: Wait for frontend service
+#=============================
+log_info "[1/4] Checking if frontend service is up (${FRONTEND_URL})..."
+
+for ((i=1; i<=TIMEOUT; i++)); do
+  STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" "$FRONTEND_URL" || true)
+  if [[ "$STATUS_CODE" == "200" ]]; then
+    log_success "Frontend service is accessible at ${FRONTEND_URL}"
+    break
+  fi
+  sleep 2
+  if [[ $i -eq $TIMEOUT ]]; then
+    fail_exit "Timeout waiting for frontend service to become ready (${TIMEOUT}s)."
+  fi
+done
+
+#=============================
+# Step 2: Run Playwright tests
+#=============================
+log_info "[2/4] Running Playwright automated tests in headless mode..."
+
+cd "$WEB_DIR"
+
+# Ensure dependencies installed
+if [ ! -d "node_modules" ]; then
+  log_warn "Dependencies not found. Installing via npm ci..."
+  npm ci
+fi
+
+log_info "Checking Playwright browsers..."
+if [ -d "node_modules/playwright" ]; then
+    log_info "Found node_modules/playwright, checking if browsers are complete..."
+    # 使用 dry-run 确认浏览器是否完整
+    if npx playwright install --dry-run | grep -q "All required browsers are installed"; then
+        log_info "All Playwright browsers are already installed, skipping installation."
+        exit 0
+    else
+        log_info "Playwright browsers incomplete, installing..."
+    fi
+else
+    log_info "Playwright browsers not found, installing..."
+    npx playwright install --with-deps > /dev/null
+fi
+
+# Clean previous reports
+rm -rf "$REPORT_DIR"
+
+# Run Playwright tests wrapped with xvfb-run to avoid GUI
+set +e  # temporarily disable exit-on-error
+env BASE_URL="$FRONTEND_URL" xvfb-run --auto-servernum npx playwright test tests/playwright --reporter=list
+TEST_RESULT=$?
+set -e  # re-enable strict mode
+
+#=============================
+# Step 3: Check test results
+#=============================
+log_info "[3/4] Checking test results..."
+
+if [[ $TEST_RESULT -eq 0 ]]; then
+  log_success "All Playwright tests passed successfully."
+else
+  log_error "Some Playwright tests failed. Please review the test report."
+fi
+
+#=============================
+# Step 4: Report generation
+#=============================
+log_info "[4/4] Checking Playwright report..."
+
+if [[ -d "$REPORT_DIR" ]]; then
+  log_success "Test report generated at: $REPORT_DIR"
+  echo "You can view it using:"
+  echo "  npx playwright show-report"
+else
+  log_warn "Report directory not found. Check Playwright execution logs."
+fi
+
+log_success "Web frontend verify finished."