完成a6000测试系统构建、部署、测试整合 (#35)

测试方案：

- lm2机器端口映射到本机：18080, 18081, 8082-8085
- 访问URL: http://localhost:18080/dashboard

![image.png](/attachments/30ed6e20-697a-4d3b-a6d3-6acccd2e9922)

![image.png](/attachments/38ef1751-0f3b-49c6-9100-f70d15617acc)

![image.png](/attachments/3be45005-9b9e-4165-8ef6-1d27405800f1)

![image.png](/attachments/eb916192-edc1-4096-8f9f-9769ab6d9039)

![image.png](/attachments/620e6efc-bd02-45ae-bba1-99a95a1b4c02)

![image.png](/attachments/986e77e7-c687-405f-a760-93282249f72f)

端到端测试通过：

![image.png](/attachments/c6e29875-4a16-4718-8b2f-368f64eb545e)

Co-authored-by: sundapeng.sdp <sundapeng@hashdata.cn>
Reviewed-on: #35
Reviewed-by: xuxt <xuxt@zgclab.edu.cn>
Reviewed-by: sundapeng <sundp@mail.zgclab.edu.cn>
Reviewed-by: huhy <husteryezi@163.com>

src/sys/tests/scripts/07_logs_send_and_assert.sh

@@ -3,9 +3,25 @@ set -euo pipefail
 
 echo "[INFO] Sending logs via node-a/node-b and asserting ES counts..."
 
+# 载入端口变量
+TEST_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")"/.. && pwd)"
+if [[ -f "$TEST_ROOT/.env" ]]; then
+  set -a; source "$TEST_ROOT/.env"; set +a
+fi
+
+# Robust count helper: tolerates 404/503 and non-JSON responses, returns integer >=0
 get_count() {
-  local idx="$1"
-  curl -s "http://localhost:9200/${idx}/_count?ignore_unavailable=true&allow_no_indices=true" | sed -E 's/.*"count":([0-9]+).*/\1/' | awk 'NF{print $0;exit} END{if(NR==0)print 0}'
+  local idx="$1"; local tmp; tmp=$(mktemp)
+  local code
+  code=$(curl -s -o "$tmp" -w "%{http_code}" "http://localhost:${ES_HTTP_PORT:-9200}/${idx}/_count?ignore_unavailable=true&allow_no_indices=true" || true)
+  if [[ "$code" == "200" ]]; then
+    local val
+    val=$(jq -r '(.count // 0) | tonumber? // 0' "$tmp" 2>/dev/null || echo 0)
+    echo "$val"
+  else
+    echo 0
+  fi
+  rm -f "$tmp"
 }
 
 train0=$(get_count "train-*")
@@ -32,11 +48,26 @@ send_logs "$node_a" "host01"
 send_logs "$node_b" "host02"
 
 echo "[INFO] Waiting for ES to ingest..."
-sleep 10
+# Proactively refresh indices (ignore errors if not created yet)
+curl -s -X POST "http://localhost:${ES_HTTP_PORT:-9200}/train-*/_refresh" >/dev/null 2>&1 || true
+curl -s -X POST "http://localhost:${ES_HTTP_PORT:-9200}/infer-*/_refresh" >/dev/null 2>&1 || true
 
-train1=$(get_count "train-*")
-infer1=$(get_count "infer-*")
-final=$((train1 + infer1))
+# Retry up to 120s for counts to increase and reach threshold (>=4)
+final=0
+threshold=4
+for attempt in {1..60}; do
+  train1=$(get_count "train-*")
+  infer1=$(get_count "infer-*")
+  final=$((train1 + infer1))
+  if (( final > base && final >= threshold )); then
+    break
+  fi
+  echo "[..] waiting ES counts increase to >=${threshold} ($attempt/60) current=${final} base=${base}"
+  # refresh indices again to speed up visibility
+  curl -s -X POST "http://localhost:${ES_HTTP_PORT:-9200}/train-*/_refresh" >/dev/null 2>&1 || true
+  curl -s -X POST "http://localhost:${ES_HTTP_PORT:-9200}/infer-*/_refresh" >/dev/null 2>&1 || true
+  sleep 2
+done
 echo "[INFO] final counts: train=${train1} infer=${infer1} total=${final}"
 
 if (( final <= base )); then
@@ -44,19 +75,20 @@ if (( final <= base )); then
   exit 1
 fi
 
+# Minimal threshold to be tolerant: expect at least 4 documents (2 train + 1 infer per node)
 if (( final < 4 )); then
   echo "[ERR] ES total below expected threshold: ${final} < 4" >&2
   exit 1
 fi
 
 # Health endpoints
-es_health=$(curl -s "http://localhost:9200/_cluster/health" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
+es_health=$(curl -s "http://localhost:${ES_HTTP_PORT:-9200}/_cluster/health" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
 if [[ "$es_health" != "green" && "$es_health" != "yellow" ]]; then
   echo "[ERR] ES health not green/yellow: $es_health" >&2
   exit 1
 fi
 
-if ! curl -fs "http://localhost:5601/api/status" >/dev/null 2>&1; then
+if ! curl -fs "http://localhost:${KIBANA_PORT:-5601}/api/status" >/dev/null 2>&1; then
   echo "[WARN] Kibana status endpoint not available"
 fi