增加sys/debug 部署测试;agent dev/user/instance元信息提取优化;sys/tests 优化 (#26)
Reviewed-on: #26 Reviewed-by: xuxt <xuxt@zgclab.edu.cn> Reviewed-by: huhy <husteryezi@163.com> Reviewed-by: sundapeng <sundp@mail.zgclab.edu.cn>
This commit was merged in pull request #26.
This commit is contained in:
73
src/sys/debug/scripts/07_logs_send_and_assert.sh
Executable file
73
src/sys/debug/scripts/07_logs_send_and_assert.sh
Executable file
@@ -0,0 +1,73 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# shellcheck source=common.sh
|
||||
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/common.sh"
|
||||
|
||||
ensure_env_file
|
||||
ensure_paths_defined
|
||||
|
||||
log "Sending logs and asserting ES counts"
|
||||
|
||||
get_count() {
|
||||
local idx="$1"
|
||||
curl -s "http://localhost:9200/${idx}/_count?ignore_unavailable=true&allow_no_indices=true" | sed -E 's/.*"count":([0-9]+).*/\1/' | awk 'NF{print $0;exit} END{if(NR==0)print 0}'
|
||||
}
|
||||
|
||||
train0=$(get_count "train-*")
|
||||
infer0=$(get_count "infer-*")
|
||||
base=$((train0 + infer0))
|
||||
log "initial counts: train=${train0} infer=${infer0} total=${base}"
|
||||
|
||||
service_id() {
|
||||
compose ps -q "$1"
|
||||
}
|
||||
|
||||
send_logs() {
|
||||
local sid="$1"; local hosttag="$2"
|
||||
docker exec "$sid" sh -lc 'mkdir -p /logs/train /logs/infer'
|
||||
docker exec "$sid" sh -lc "ts=\
|
||||
\$(date '+%F %T'); echo \"\$ts INFO [$hosttag] training step=1 loss=1.23 model=bert\" >> /logs/train/train-demo.log"
|
||||
docker exec "$sid" sh -lc "ts=\
|
||||
\$(date '+%F %T'); echo \"\$ts INFO [$hosttag] training step=2 loss=1.10 model=bert\" >> /logs/train/train-demo.log"
|
||||
docker exec "$sid" sh -lc "ts=\
|
||||
\$(date '+%F %T'); echo \"\$ts WARN [$hosttag] inference slow on batch=2 latency=1.9s\" >> /logs/infer/infer-demo.log"
|
||||
}
|
||||
|
||||
CID_A="$(service_id node-a)"
|
||||
CID_B="$(service_id node-b)"
|
||||
|
||||
[[ -n "$CID_A" && -n "$CID_B" ]] || { echo "[ERR] node containers not found" >&2; exit 1; }
|
||||
|
||||
send_logs "$CID_A" "host01"
|
||||
send_logs "$CID_B" "host02"
|
||||
|
||||
log "Waiting for ES to ingest"
|
||||
sleep 10
|
||||
|
||||
train1=$(get_count "train-*")
|
||||
infer1=$(get_count "infer-*")
|
||||
final=$((train1 + infer1))
|
||||
log "final counts: train=${train1} infer=${infer1} total=${final}"
|
||||
|
||||
if (( final <= base )); then
|
||||
echo "[ERR] ES total did not increase (${base} -> ${final})" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if (( final < 4 )); then
|
||||
echo "[ERR] ES total below expected threshold: ${final} < 4" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
es_health=$(curl -s "http://localhost:9200/_cluster/health" | grep -o '"status":"[^"]*"' | cut -d'"' -f4)
|
||||
if [[ "$es_health" != "green" && "$es_health" != "yellow" ]]; then
|
||||
echo "[ERR] ES health not green/yellow: $es_health" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! curl -fs "http://localhost:5601/api/status" >/dev/null 2>&1; then
|
||||
echo "[WARN] Kibana status endpoint not available"
|
||||
fi
|
||||
|
||||
log "ES counts increased and services healthy"
|
||||
Reference in New Issue
Block a user