20260721 metrics 服务新增 CCR rpki-client 格式检查指标与 Grafana 面板（#125）
This commit is contained in:
parent
4074323302
commit
f86fbd4939
@ -1,6 +1,7 @@
|
|||||||
target/
|
target/
|
||||||
.git/
|
.git/
|
||||||
.gitignore
|
.gitignore
|
||||||
|
docker/
|
||||||
perf.*
|
perf.*
|
||||||
**/* copy.excalidraw
|
**/* copy.excalidraw
|
||||||
ui/rpki-explorer/node_modules/
|
ui/rpki-explorer/node_modules/
|
||||||
|
|||||||
@ -88,6 +88,12 @@ RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS=2000
|
|||||||
METRICS_INSTANCE=__PACKAGE_ARCH__-installer
|
METRICS_INSTANCE=__PACKAGE_ARCH__-installer
|
||||||
METRICS_PORT=9556
|
METRICS_PORT=9556
|
||||||
METRICS_POLL_SECS=10
|
METRICS_POLL_SECS=10
|
||||||
|
# CCR rpki-client format check: the artifact-metrics container runs
|
||||||
|
# `rpki-client -f result.ccr` once for every newly completed run and exports
|
||||||
|
# ours_rp_ccr_format_check_* metrics. The binary is packaged inside the
|
||||||
|
# metrics image; adjust only when deliberately testing another binary.
|
||||||
|
CCR_CHECK_BIN=/opt/ours-rp/bin/rpki-client
|
||||||
|
CCR_CHECK_TIMEOUT_SECS=120
|
||||||
# Optional external RTR report directory produced by a separately deployed RTR service.
|
# Optional external RTR report directory produced by a separately deployed RTR service.
|
||||||
# Default points to an installer-managed empty fallback directory. To enable real RTR
|
# Default points to an installer-managed empty fallback directory. To enable real RTR
|
||||||
# metrics, set this to the host-side report directory, for example:
|
# metrics, set this to the host-side report directory, for example:
|
||||||
|
|||||||
@ -39,6 +39,10 @@ services:
|
|||||||
- ${METRICS_POLL_SECS:-10}
|
- ${METRICS_POLL_SECS:-10}
|
||||||
- --instance
|
- --instance
|
||||||
- ${METRICS_INSTANCE:-package-installer}
|
- ${METRICS_INSTANCE:-package-installer}
|
||||||
|
- --ccr-check-bin
|
||||||
|
- ${CCR_CHECK_BIN:-/opt/ours-rp/bin/rpki-client}
|
||||||
|
- --ccr-check-timeout-secs
|
||||||
|
- ${CCR_CHECK_TIMEOUT_SECS:-120}
|
||||||
ports:
|
ports:
|
||||||
- "${METRICS_PORT:-9556}:9556"
|
- "${METRICS_PORT:-9556}:9556"
|
||||||
volumes:
|
volumes:
|
||||||
|
|||||||
@ -850,6 +850,264 @@
|
|||||||
],
|
],
|
||||||
"title": "Fresh PP / Object Counts by Run",
|
"title": "Fresh PP / Object Counts by Run",
|
||||||
"type": "timeseries"
|
"type": "timeseries"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "Prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"decimals": 0,
|
||||||
|
"unit": "none",
|
||||||
|
"mappings": [
|
||||||
|
{
|
||||||
|
"type": "value",
|
||||||
|
"options": {
|
||||||
|
"1": {
|
||||||
|
"text": "PASS",
|
||||||
|
"color": "green"
|
||||||
|
},
|
||||||
|
"0": {
|
||||||
|
"text": "FAIL",
|
||||||
|
"color": "red"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 56,
|
||||||
|
"w": 6,
|
||||||
|
"h": 4
|
||||||
|
},
|
||||||
|
"id": 19,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"textMode": "auto",
|
||||||
|
"wideLayout": true
|
||||||
|
},
|
||||||
|
"pluginVersion": "11.3.1",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "max(ours_rp_ccr_format_check_last_success and on (exported_instance) topk(1, ours_rp_run_sequence))",
|
||||||
|
"legendFormat": "CCR check",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "Latest CCR Format Check",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "Prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"decimals": 0,
|
||||||
|
"unit": "none"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 6,
|
||||||
|
"y": 56,
|
||||||
|
"w": 6,
|
||||||
|
"h": 4
|
||||||
|
},
|
||||||
|
"id": 20,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"textMode": "auto",
|
||||||
|
"wideLayout": true
|
||||||
|
},
|
||||||
|
"pluginVersion": "11.3.1",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "max(ours_rp_ccr_format_check_last_run_sequence and on (exported_instance) topk(1, ours_rp_run_sequence))",
|
||||||
|
"legendFormat": "run seq",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "CCR Checked Run Sequence",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "Prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"decimals": 1,
|
||||||
|
"unit": "s"
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 12,
|
||||||
|
"y": 56,
|
||||||
|
"w": 6,
|
||||||
|
"h": 4
|
||||||
|
},
|
||||||
|
"id": 21,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"textMode": "auto",
|
||||||
|
"wideLayout": true
|
||||||
|
},
|
||||||
|
"pluginVersion": "11.3.1",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "max(ours_rp_ccr_format_check_last_duration_seconds and on (exported_instance) topk(1, ours_rp_run_sequence))",
|
||||||
|
"legendFormat": "duration",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "CCR Check Duration",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "Prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"decimals": 0,
|
||||||
|
"unit": "none",
|
||||||
|
"thresholds": {
|
||||||
|
"mode": "absolute",
|
||||||
|
"steps": [
|
||||||
|
{
|
||||||
|
"color": "green",
|
||||||
|
"value": null
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"color": "red",
|
||||||
|
"value": 1
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 18,
|
||||||
|
"y": 56,
|
||||||
|
"w": 6,
|
||||||
|
"h": 4
|
||||||
|
},
|
||||||
|
"id": 22,
|
||||||
|
"options": {
|
||||||
|
"colorMode": "value",
|
||||||
|
"graphMode": "area",
|
||||||
|
"justifyMode": "auto",
|
||||||
|
"orientation": "auto",
|
||||||
|
"reduceOptions": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull"
|
||||||
|
],
|
||||||
|
"fields": "",
|
||||||
|
"values": false
|
||||||
|
},
|
||||||
|
"textMode": "auto",
|
||||||
|
"wideLayout": true
|
||||||
|
},
|
||||||
|
"pluginVersion": "11.3.1",
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "max(ours_rp_ccr_format_check_consecutive_failures and on (exported_instance) topk(1, ours_rp_run_sequence))",
|
||||||
|
"legendFormat": "failures",
|
||||||
|
"refId": "A"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "CCR Check Consecutive Failures",
|
||||||
|
"type": "stat"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"datasource": {
|
||||||
|
"type": "prometheus",
|
||||||
|
"uid": "Prometheus"
|
||||||
|
},
|
||||||
|
"fieldConfig": {
|
||||||
|
"defaults": {
|
||||||
|
"unit": "none",
|
||||||
|
"decimals": 0,
|
||||||
|
"min": 0
|
||||||
|
},
|
||||||
|
"overrides": []
|
||||||
|
},
|
||||||
|
"gridPos": {
|
||||||
|
"x": 0,
|
||||||
|
"y": 60,
|
||||||
|
"w": 24,
|
||||||
|
"h": 8
|
||||||
|
},
|
||||||
|
"id": 23,
|
||||||
|
"options": {
|
||||||
|
"legend": {
|
||||||
|
"calcs": [
|
||||||
|
"lastNotNull",
|
||||||
|
"max"
|
||||||
|
],
|
||||||
|
"displayMode": "table",
|
||||||
|
"placement": "bottom",
|
||||||
|
"showLegend": true
|
||||||
|
},
|
||||||
|
"tooltip": {
|
||||||
|
"mode": "multi",
|
||||||
|
"sort": "none"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"targets": [
|
||||||
|
{
|
||||||
|
"expr": "sum by (result) (ours_rp_ccr_format_check_total)",
|
||||||
|
"legendFormat": "{{result}} checks",
|
||||||
|
"refId": "A"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"expr": "max(ours_rp_ccr_format_check_last_success and on (exported_instance) topk(1, ours_rp_run_sequence))",
|
||||||
|
"legendFormat": "latest check ok",
|
||||||
|
"refId": "B"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"title": "CCR Format Check Results",
|
||||||
|
"type": "timeseries"
|
||||||
}
|
}
|
||||||
],
|
],
|
||||||
"refresh": "5s",
|
"refresh": "5s",
|
||||||
|
|||||||
@ -125,6 +125,12 @@ If a separately deployed RTR service on the same host continuously writes `rtr-s
|
|||||||
|
|
||||||
The command accepts only an absolute path on the same host, validates the candidate Compose mount, and atomically updates `.env`. It recreates only `artifact-metrics`, Prometheus, and Grafana; it does not restart or recreate `ours-rp-soak`. The `artifact-metrics` container mounts the directory read-only and reads it through `RPKI_METRICS_RTR_REPORT_DIR`, producing `ours_rp_rtr_*` metrics. By default, `RTR_REPORT_DIR` points at an empty fallback directory under the installer data root, so the metrics container also starts cleanly without an RTR service.
|
The command accepts only an absolute path on the same host, validates the candidate Compose mount, and atomically updates `.env`. It recreates only `artifact-metrics`, Prometheus, and Grafana; it does not restart or recreate `ours-rp-soak`. The `artifact-metrics` container mounts the directory read-only and reads it through `RPKI_METRICS_RTR_REPORT_DIR`, producing `ours_rp_rtr_*` metrics. By default, `RTR_REPORT_DIR` points at an empty fallback directory under the installer data root, so the metrics container also starts cleanly without an RTR service.
|
||||||
|
|
||||||
|
## CCR artifact format check metrics
|
||||||
|
|
||||||
|
The metrics image ships the official `rpki-client 9.8`. For every newly completed run, `artifact-metrics` runs `rpki-client -f result.ccr` once and exports `ours_rp_ccr_format_check_*` metrics (cumulative pass/ccr_parse_error/tool_error counters, latest check result, check duration, consecutive failures). The Grafana `Ours RP Soak Overview` dashboard shows them in a dedicated panel row. On service start only the latest existing run is checked; older history is not backfilled.
|
||||||
|
|
||||||
|
Related environment variables (normally no need to change): `CCR_CHECK_BIN` (defaults to the in-image `/opt/ours-rp/bin/rpki-client`) and `CCR_CHECK_TIMEOUT_SECS` (default 120 seconds).
|
||||||
|
|
||||||
## Data Directory
|
## Data Directory
|
||||||
|
|
||||||
Default host directory:
|
Default host directory:
|
||||||
|
|||||||
@ -125,6 +125,12 @@ GRAFANA_ADMIN_PASSWORD=admin
|
|||||||
|
|
||||||
该命令只接受本机绝对路径，会先校验候选 Compose 挂载，再原子更新 `.env`。随后只重建 `artifact-metrics`、Prometheus 和 Grafana，不重启或重建 `ours-rp-soak`。`artifact-metrics` 以只读方式挂载该目录，并通过 `RPKI_METRICS_RTR_REPORT_DIR` 读取 report，输出 `ours_rp_rtr_*` 指标。默认 `RTR_REPORT_DIR` 指向安装包数据目录下的空 fallback 目录，因此未接入 RTR 服务时 metrics 容器也能稳定启动。
|
该命令只接受本机绝对路径，会先校验候选 Compose 挂载，再原子更新 `.env`。随后只重建 `artifact-metrics`、Prometheus 和 Grafana，不重启或重建 `ours-rp-soak`。`artifact-metrics` 以只读方式挂载该目录，并通过 `RPKI_METRICS_RTR_REPORT_DIR` 读取 report，输出 `ours_rp_rtr_*` 指标。默认 `RTR_REPORT_DIR` 指向安装包数据目录下的空 fallback 目录，因此未接入 RTR 服务时 metrics 容器也能稳定启动。
|
||||||
|
|
||||||
|
## CCR 产物格式检查监控
|
||||||
|
|
||||||
|
metrics 镜像内置官方 `rpki-client 9.8`。`artifact-metrics` 会对每个新完成 run 的 `result.ccr` 执行一次 `rpki-client -f` 格式检查，并输出 `ours_rp_ccr_format_check_*` 指标（累计 pass/ccr_parse_error/tool_error、最近检查结果、检查耗时、连续失败数）。Grafana `Ours RP Soak Overview` 底部有对应面板。服务启动时只补查最新的一个历史 run，不回扫全部历史。
|
||||||
|
|
||||||
|
相关环境变量（通常无需修改）：`CCR_CHECK_BIN`（默认为镜像内 `/opt/ours-rp/bin/rpki-client`）、`CCR_CHECK_TIMEOUT_SECS`（默认 120 秒）。
|
||||||
|
|
||||||
## 数据目录
|
## 数据目录
|
||||||
|
|
||||||
默认宿主机目录：
|
默认宿主机目录：
|
||||||
|
|||||||
@ -756,6 +756,18 @@ verify_metrics_image() {
|
|||||||
log "verifying metrics image $METRICS_IMAGE on $METRICS_PLATFORM"
|
log "verifying metrics image $METRICS_IMAGE on $METRICS_PLATFORM"
|
||||||
verify_image_platform "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics"
|
verify_image_platform "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics"
|
||||||
verify_image_usage_output "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics" /opt/ours-rp/bin/rpki_artifact_metrics --help
|
verify_image_usage_output "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics" /opt/ours-rp/bin/rpki_artifact_metrics --help
|
||||||
|
verify_metrics_image_rpki_client
|
||||||
|
}
|
||||||
|
|
||||||
|
verify_metrics_image_rpki_client() {
|
||||||
|
local output
|
||||||
|
local status
|
||||||
|
set +e
|
||||||
|
output="$(docker run --rm --platform "$METRICS_PLATFORM" "$METRICS_IMAGE" /opt/ours-rp/bin/rpki-client -V 2>&1)"
|
||||||
|
status=$?
|
||||||
|
set -e
|
||||||
|
[[ "$status" == "0" ]] || die "metrics image rpki-client -V failed: image=$METRICS_IMAGE exit=$status output=$output"
|
||||||
|
grep -q "rpki-client" <<<"$output" || die "metrics image rpki-client -V unexpected output: $output"
|
||||||
}
|
}
|
||||||
|
|
||||||
verify_image_platform() {
|
verify_image_platform() {
|
||||||
|
|||||||
@ -85,27 +85,66 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
|
|||||||
mkdir -p /build-out/bin; \
|
mkdir -p /build-out/bin; \
|
||||||
cp "target/$target_triple/release/rpki_artifact_metrics" /build-out/bin/
|
cp "target/$target_triple/release/rpki_artifact_metrics" /build-out/bin/
|
||||||
|
|
||||||
|
FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS rpki-client-builder
|
||||||
|
|
||||||
|
ARG RPKI_CLIENT_VERSION=9.8
|
||||||
|
ARG RPKI_CLIENT_SHA256=42920aac5afd0996173fb9f7848691a2c49dd234e4f10478808c1aa475620861
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
ca-certificates \
|
||||||
|
curl \
|
||||||
|
libexpat1-dev \
|
||||||
|
libssl-dev \
|
||||||
|
libtls-dev \
|
||||||
|
pkg-config \
|
||||||
|
rsync \
|
||||||
|
zlib1g-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& useradd --system --no-create-home --shell /usr/sbin/nologin _rpki-client
|
||||||
|
|
||||||
|
RUN set -eux; \
|
||||||
|
curl -fsSL "https://ftp.openbsd.org/pub/OpenBSD/rpki-client/rpki-client-${RPKI_CLIENT_VERSION}.tar.gz" -o /tmp/rpki-client.tar.gz; \
|
||||||
|
echo "${RPKI_CLIENT_SHA256} /tmp/rpki-client.tar.gz" | sha256sum -c -; \
|
||||||
|
tar -xzf /tmp/rpki-client.tar.gz -C /tmp; \
|
||||||
|
cd "/tmp/rpki-client-${RPKI_CLIENT_VERSION}"; \
|
||||||
|
./configure --prefix=/opt/rpki-client; \
|
||||||
|
make -j"$(nproc)"; \
|
||||||
|
make install; \
|
||||||
|
/opt/rpki-client/sbin/rpki-client -V
|
||||||
|
|
||||||
FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime
|
FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime
|
||||||
|
|
||||||
RUN apt-get update \
|
RUN apt-get update \
|
||||||
&& apt-get install -y --no-install-recommends \
|
&& apt-get install -y --no-install-recommends \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
|
libexpat1 \
|
||||||
|
libssl3 \
|
||||||
|
libtls26 \
|
||||||
tzdata \
|
tzdata \
|
||||||
&& rm -rf /var/lib/apt/lists/*
|
zlib1g \
|
||||||
|
&& rm -rf /var/lib/apt/lists/* \
|
||||||
|
&& useradd --system --no-create-home --shell /usr/sbin/nologin _rpki-client \
|
||||||
|
&& mkdir -p /opt/rpki-client/var/cache/rpki-client
|
||||||
|
|
||||||
WORKDIR /opt/ours-rp
|
WORKDIR /opt/ours-rp
|
||||||
|
|
||||||
COPY --from=builder /build-out/bin/ /opt/ours-rp/bin/
|
COPY --from=builder /build-out/bin/ /opt/ours-rp/bin/
|
||||||
|
COPY --from=rpki-client-builder /opt/rpki-client/ /opt/rpki-client/
|
||||||
|
COPY --from=rpki-client-builder /opt/rpki-client/sbin/rpki-client /opt/ours-rp/bin/rpki-client
|
||||||
|
|
||||||
ARG SOURCE_COMMIT=unknown
|
ARG SOURCE_COMMIT=unknown
|
||||||
ARG SOURCE_DIRTY=unknown
|
ARG SOURCE_DIRTY=unknown
|
||||||
ARG BUILD_TIMESTAMP_UTC=unknown
|
ARG BUILD_TIMESTAMP_UTC=unknown
|
||||||
|
ARG RPKI_CLIENT_VERSION=9.8
|
||||||
|
|
||||||
LABEL org.opencontainers.image.title="ours-rp-metrics" \
|
LABEL org.opencontainers.image.title="ours-rp-metrics" \
|
||||||
org.opencontainers.image.description="Ours RP artifact metrics image for multi-arch Docker Compose deployment" \
|
org.opencontainers.image.description="Ours RP artifact metrics image for multi-arch Docker Compose deployment" \
|
||||||
org.opencontainers.image.revision="${SOURCE_COMMIT}" \
|
org.opencontainers.image.revision="${SOURCE_COMMIT}" \
|
||||||
org.opencontainers.image.created="${BUILD_TIMESTAMP_UTC}" \
|
org.opencontainers.image.created="${BUILD_TIMESTAMP_UTC}" \
|
||||||
org.opencontainers.image.source-dirty="${SOURCE_DIRTY}"
|
org.opencontainers.image.source-dirty="${SOURCE_DIRTY}" \
|
||||||
|
org.opencontainers.image.rpki-client.version="${RPKI_CLIENT_VERSION}"
|
||||||
|
|
||||||
RUN chmod +x /opt/ours-rp/bin/* \
|
RUN chmod +x /opt/ours-rp/bin/* \
|
||||||
&& mkdir -p /var/lib/ours-rp/state /var/lib/ours-rp/runs /var/lib/ours-rp/logs
|
&& mkdir -p /var/lib/ours-rp/state /var/lib/ours-rp/runs /var/lib/ours-rp/logs
|
||||||
|
|||||||
@ -3,6 +3,7 @@ use std::fs;
|
|||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::net::{TcpListener, TcpStream};
|
use std::net::{TcpListener, TcpStream};
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::process::{Command, Stdio};
|
||||||
use std::sync::{Arc, RwLock};
|
use std::sync::{Arc, RwLock};
|
||||||
use std::thread;
|
use std::thread;
|
||||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||||
@ -26,10 +27,13 @@ struct Args {
|
|||||||
once: bool,
|
once: bool,
|
||||||
out_metrics: Option<PathBuf>,
|
out_metrics: Option<PathBuf>,
|
||||||
out_status: Option<PathBuf>,
|
out_status: Option<PathBuf>,
|
||||||
|
ccr_check_bin: Option<PathBuf>,
|
||||||
|
ccr_check_timeout_secs: u64,
|
||||||
|
ccr_check_max_per_scan: u64,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn usage() -> &'static str {
|
fn usage() -> &'static str {
|
||||||
"Usage: rpki_artifact_metrics --run-root <path> [--listen <addr:port>] [--poll-secs <n>] [--instance <name>] [--once] [--out-metrics <path>] [--out-status <path>]"
|
"Usage: rpki_artifact_metrics --run-root <path> [--listen <addr:port>] [--poll-secs <n>] [--instance <name>] [--once] [--out-metrics <path>] [--out-status <path>] [--ccr-check-bin <path>] [--ccr-check-timeout-secs <n>] [--ccr-check-max-per-scan <n>]"
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn main_entry() -> Result<(), String> {
|
pub fn main_entry() -> Result<(), String> {
|
||||||
@ -38,8 +42,16 @@ pub fn main_entry() -> Result<(), String> {
|
|||||||
|
|
||||||
fn real_main() -> Result<(), String> {
|
fn real_main() -> Result<(), String> {
|
||||||
let args = parse_args(&std::env::args().collect::<Vec<_>>())?;
|
let args = parse_args(&std::env::args().collect::<Vec<_>>())?;
|
||||||
|
let ccr_check_config = CcrCheckConfig::from_args(&args);
|
||||||
|
let rtr_report_dir = configured_rtr_report_dir();
|
||||||
if args.once {
|
if args.once {
|
||||||
let snapshot = scan_run_root(&args.run_root, &args.instance)?;
|
let snapshot = scan_run_root_incremental_with_rtr_dir(
|
||||||
|
&args.run_root,
|
||||||
|
&args.instance,
|
||||||
|
None,
|
||||||
|
rtr_report_dir.as_deref(),
|
||||||
|
&ccr_check_config,
|
||||||
|
)?;
|
||||||
let metrics = render_metrics(&snapshot);
|
let metrics = render_metrics(&snapshot);
|
||||||
let status = render_status_json(&snapshot)?;
|
let status = render_status_json(&snapshot)?;
|
||||||
if let Some(path) = args.out_metrics.as_ref() {
|
if let Some(path) = args.out_metrics.as_ref() {
|
||||||
@ -53,7 +65,13 @@ fn real_main() -> Result<(), String> {
|
|||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let shared = Arc::new(RwLock::new(scan_run_root(&args.run_root, &args.instance)?));
|
let shared = Arc::new(RwLock::new(scan_run_root_incremental_with_rtr_dir(
|
||||||
|
&args.run_root,
|
||||||
|
&args.instance,
|
||||||
|
None,
|
||||||
|
rtr_report_dir.as_deref(),
|
||||||
|
&ccr_check_config,
|
||||||
|
)?));
|
||||||
let scanner = Arc::clone(&shared);
|
let scanner = Arc::clone(&shared);
|
||||||
let run_root = args.run_root.clone();
|
let run_root = args.run_root.clone();
|
||||||
let instance = args.instance.clone();
|
let instance = args.instance.clone();
|
||||||
@ -62,8 +80,13 @@ fn real_main() -> Result<(), String> {
|
|||||||
loop {
|
loop {
|
||||||
thread::sleep(Duration::from_secs(poll_secs));
|
thread::sleep(Duration::from_secs(poll_secs));
|
||||||
let previous_snapshot = scanner.read().expect("metrics lock poisoned").clone();
|
let previous_snapshot = scanner.read().expect("metrics lock poisoned").clone();
|
||||||
let next =
|
let next = match scan_run_root_incremental_with_rtr_dir(
|
||||||
match scan_run_root_incremental(&run_root, &instance, Some(&previous_snapshot)) {
|
&run_root,
|
||||||
|
&instance,
|
||||||
|
Some(&previous_snapshot),
|
||||||
|
rtr_report_dir.as_deref(),
|
||||||
|
&ccr_check_config,
|
||||||
|
) {
|
||||||
Ok(snapshot) => snapshot,
|
Ok(snapshot) => snapshot,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
let mut previous = scanner.write().expect("metrics lock poisoned");
|
let mut previous = scanner.write().expect("metrics lock poisoned");
|
||||||
@ -91,6 +114,9 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
|||||||
let mut once = false;
|
let mut once = false;
|
||||||
let mut out_metrics = None;
|
let mut out_metrics = None;
|
||||||
let mut out_status = None;
|
let mut out_status = None;
|
||||||
|
let mut ccr_check_bin = None;
|
||||||
|
let mut ccr_check_timeout_secs = 120u64;
|
||||||
|
let mut ccr_check_max_per_scan = 2u64;
|
||||||
let mut index = 1usize;
|
let mut index = 1usize;
|
||||||
while index < argv.len() {
|
while index < argv.len() {
|
||||||
match argv[index].as_str() {
|
match argv[index].as_str() {
|
||||||
@ -122,6 +148,24 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
|||||||
index += 1;
|
index += 1;
|
||||||
out_status = Some(PathBuf::from(value_at(argv, index, "--out-status")?));
|
out_status = Some(PathBuf::from(value_at(argv, index, "--out-status")?));
|
||||||
}
|
}
|
||||||
|
"--ccr-check-bin" => {
|
||||||
|
index += 1;
|
||||||
|
ccr_check_bin = Some(PathBuf::from(value_at(argv, index, "--ccr-check-bin")?));
|
||||||
|
}
|
||||||
|
"--ccr-check-timeout-secs" => {
|
||||||
|
index += 1;
|
||||||
|
let value = value_at(argv, index, "--ccr-check-timeout-secs")?;
|
||||||
|
ccr_check_timeout_secs = value
|
||||||
|
.parse::<u64>()
|
||||||
|
.map_err(|_| format!("invalid --ccr-check-timeout-secs: {value}"))?;
|
||||||
|
}
|
||||||
|
"--ccr-check-max-per-scan" => {
|
||||||
|
index += 1;
|
||||||
|
let value = value_at(argv, index, "--ccr-check-max-per-scan")?;
|
||||||
|
ccr_check_max_per_scan = value
|
||||||
|
.parse::<u64>()
|
||||||
|
.map_err(|_| format!("invalid --ccr-check-max-per-scan: {value}"))?;
|
||||||
|
}
|
||||||
"-h" | "--help" => return Err(usage().to_string()),
|
"-h" | "--help" => return Err(usage().to_string()),
|
||||||
other => return Err(format!("unknown argument: {other}\n{}", usage())),
|
other => return Err(format!("unknown argument: {other}\n{}", usage())),
|
||||||
}
|
}
|
||||||
@ -135,6 +179,9 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
|
|||||||
once,
|
once,
|
||||||
out_metrics,
|
out_metrics,
|
||||||
out_status,
|
out_status,
|
||||||
|
ccr_check_bin,
|
||||||
|
ccr_check_timeout_secs,
|
||||||
|
ccr_check_max_per_scan,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -161,9 +208,67 @@ struct MetricsSnapshot {
|
|||||||
top_pp_by_sync_duration: Vec<TopPublicationPoint>,
|
top_pp_by_sync_duration: Vec<TopPublicationPoint>,
|
||||||
cir: Option<CirMetrics>,
|
cir: Option<CirMetrics>,
|
||||||
ccr: Option<CcrMetrics>,
|
ccr: Option<CcrMetrics>,
|
||||||
|
ccr_check: CcrCheckMetrics,
|
||||||
rtr: RtrMetrics,
|
rtr: RtrMetrics,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Default, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
struct CcrCheckMetrics {
|
||||||
|
enabled: bool,
|
||||||
|
up: bool,
|
||||||
|
bin: Option<String>,
|
||||||
|
pass_total: u64,
|
||||||
|
ccr_parse_error_total: u64,
|
||||||
|
tool_error_total: u64,
|
||||||
|
consecutive_failures: u64,
|
||||||
|
last: Option<CcrCheckLastResult>,
|
||||||
|
checked_run_dirs: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize)]
|
||||||
|
#[serde(rename_all = "camelCase")]
|
||||||
|
struct CcrCheckLastResult {
|
||||||
|
run_dir: String,
|
||||||
|
run_seq: Option<u64>,
|
||||||
|
outcome: String,
|
||||||
|
duration_seconds: f64,
|
||||||
|
checked_at_seconds: f64,
|
||||||
|
detail: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug)]
|
||||||
|
struct CcrCheckConfig {
|
||||||
|
bin: Option<PathBuf>,
|
||||||
|
timeout_secs: u64,
|
||||||
|
max_per_scan: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl CcrCheckConfig {
|
||||||
|
fn disabled() -> Self {
|
||||||
|
Self {
|
||||||
|
bin: None,
|
||||||
|
timeout_secs: 120,
|
||||||
|
max_per_scan: 2,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn from_args(args: &Args) -> Self {
|
||||||
|
let bin = args.ccr_check_bin.clone().or_else(|| {
|
||||||
|
std::env::var("RPKI_METRICS_CCR_CHECK_BIN")
|
||||||
|
.ok()
|
||||||
|
.map(|value| value.trim().to_string())
|
||||||
|
.filter(|value| !value.is_empty())
|
||||||
|
.map(PathBuf::from)
|
||||||
|
});
|
||||||
|
Self {
|
||||||
|
bin,
|
||||||
|
timeout_secs: args.ccr_check_timeout_secs,
|
||||||
|
max_per_scan: args.ccr_check_max_per_scan,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, Default, Serialize)]
|
#[derive(Clone, Debug, Default, Serialize)]
|
||||||
#[serde(rename_all = "camelCase")]
|
#[serde(rename_all = "camelCase")]
|
||||||
struct ServiceMetrics {
|
struct ServiceMetrics {
|
||||||
@ -493,21 +598,13 @@ struct RunRecord {
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn scan_run_root(input_root: &Path, instance: &str) -> Result<MetricsSnapshot, String> {
|
fn scan_run_root(input_root: &Path, instance: &str) -> Result<MetricsSnapshot, String> {
|
||||||
let rtr_report_dir = configured_rtr_report_dir();
|
|
||||||
scan_run_root_incremental_with_rtr_dir(input_root, instance, None, rtr_report_dir.as_deref())
|
|
||||||
}
|
|
||||||
|
|
||||||
fn scan_run_root_incremental(
|
|
||||||
input_root: &Path,
|
|
||||||
instance: &str,
|
|
||||||
previous: Option<&MetricsSnapshot>,
|
|
||||||
) -> Result<MetricsSnapshot, String> {
|
|
||||||
let rtr_report_dir = configured_rtr_report_dir();
|
let rtr_report_dir = configured_rtr_report_dir();
|
||||||
scan_run_root_incremental_with_rtr_dir(
|
scan_run_root_incremental_with_rtr_dir(
|
||||||
input_root,
|
input_root,
|
||||||
instance,
|
instance,
|
||||||
previous,
|
None,
|
||||||
rtr_report_dir.as_deref(),
|
rtr_report_dir.as_deref(),
|
||||||
|
&CcrCheckConfig::disabled(),
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -516,6 +613,7 @@ fn scan_run_root_incremental_with_rtr_dir(
|
|||||||
instance: &str,
|
instance: &str,
|
||||||
previous: Option<&MetricsSnapshot>,
|
previous: Option<&MetricsSnapshot>,
|
||||||
rtr_report_dir: Option<&Path>,
|
rtr_report_dir: Option<&Path>,
|
||||||
|
ccr_check_config: &CcrCheckConfig,
|
||||||
) -> Result<MetricsSnapshot, String> {
|
) -> Result<MetricsSnapshot, String> {
|
||||||
let started = Instant::now();
|
let started = Instant::now();
|
||||||
let runs_root = resolve_runs_root(input_root);
|
let runs_root = resolve_runs_root(input_root);
|
||||||
@ -570,6 +668,7 @@ fn scan_run_root_incremental_with_rtr_dir(
|
|||||||
}
|
}
|
||||||
|
|
||||||
snapshot.rtr = scan_rtr_reports(rtr_report_dir, previous.map(|snapshot| &snapshot.rtr));
|
snapshot.rtr = scan_rtr_reports(rtr_report_dir, previous.map(|snapshot| &snapshot.rtr));
|
||||||
|
run_pending_ccr_checks(&records, previous, ccr_check_config, &mut snapshot);
|
||||||
snapshot.service.last_scan_timestamp_seconds = unix_now_seconds();
|
snapshot.service.last_scan_timestamp_seconds = unix_now_seconds();
|
||||||
snapshot.service.last_scan_duration_seconds = started.elapsed().as_secs_f64();
|
snapshot.service.last_scan_duration_seconds = started.elapsed().as_secs_f64();
|
||||||
snapshot.service.last_reload_success =
|
snapshot.service.last_reload_success =
|
||||||
@ -975,6 +1074,218 @@ fn consecutive_failures(records: &[RunRecord]) -> u64 {
|
|||||||
count
|
count
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const MAX_CHECKED_RUN_DIRS: usize = 512;
|
||||||
|
|
||||||
|
fn run_pending_ccr_checks(
|
||||||
|
records: &[RunRecord],
|
||||||
|
previous: Option<&MetricsSnapshot>,
|
||||||
|
config: &CcrCheckConfig,
|
||||||
|
snapshot: &mut MetricsSnapshot,
|
||||||
|
) {
|
||||||
|
let mut check = previous
|
||||||
|
.map(|prev| prev.ccr_check.clone())
|
||||||
|
.unwrap_or_default();
|
||||||
|
check.enabled = config.bin.is_some();
|
||||||
|
check.bin = config.bin.as_ref().map(|bin| bin.display().to_string());
|
||||||
|
check.up = false;
|
||||||
|
|
||||||
|
let Some(bin) = config.bin.as_ref() else {
|
||||||
|
snapshot.ccr_check = check;
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if !bin.is_file() {
|
||||||
|
snapshot
|
||||||
|
.service
|
||||||
|
.parse_errors
|
||||||
|
.push(format!("ccr check bin is not a file: {}", bin.display()));
|
||||||
|
snapshot.ccr_check = check;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
check.up = true;
|
||||||
|
|
||||||
|
let mut pending: Vec<&RunRecord> = records
|
||||||
|
.iter()
|
||||||
|
.filter(|record| record.status == "success")
|
||||||
|
.filter(|record| record.path.join("result.ccr").is_file())
|
||||||
|
.filter(|record| {
|
||||||
|
let dir = record.path.display().to_string();
|
||||||
|
!check.checked_run_dirs.iter().any(|seen| seen == &dir)
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if previous.is_none() {
|
||||||
|
// On service start only the latest existing run is checked; older
|
||||||
|
// history is marked as seen without executing checks.
|
||||||
|
let latest_path = pending.last().map(|record| record.path.clone());
|
||||||
|
for record in &pending {
|
||||||
|
if Some(&record.path) != latest_path.as_ref() {
|
||||||
|
check
|
||||||
|
.checked_run_dirs
|
||||||
|
.push(record.path.display().to_string());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
pending.retain(|record| Some(&record.path) == latest_path.as_ref());
|
||||||
|
}
|
||||||
|
|
||||||
|
for record in pending
|
||||||
|
.into_iter()
|
||||||
|
.take(config.max_per_scan.max(1) as usize)
|
||||||
|
{
|
||||||
|
let ccr_path = record.path.join("result.ccr");
|
||||||
|
let outcome = execute_ccr_check(bin, &ccr_path, Duration::from_secs(config.timeout_secs));
|
||||||
|
let run_seq = record
|
||||||
|
.summary
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| json_u64(v, &["runSeq"]))
|
||||||
|
.or_else(|| {
|
||||||
|
record
|
||||||
|
.meta
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|v| json_u64(v, &["run_index"]))
|
||||||
|
});
|
||||||
|
let passed = outcome.outcome == "pass";
|
||||||
|
match outcome.outcome.as_str() {
|
||||||
|
"pass" => check.pass_total += 1,
|
||||||
|
"ccr_parse_error" => check.ccr_parse_error_total += 1,
|
||||||
|
_ => check.tool_error_total += 1,
|
||||||
|
}
|
||||||
|
check.consecutive_failures = if passed {
|
||||||
|
0
|
||||||
|
} else {
|
||||||
|
check.consecutive_failures + 1
|
||||||
|
};
|
||||||
|
check.last = Some(CcrCheckLastResult {
|
||||||
|
run_dir: record.path.display().to_string(),
|
||||||
|
run_seq,
|
||||||
|
outcome: outcome.outcome,
|
||||||
|
duration_seconds: outcome.duration_seconds,
|
||||||
|
checked_at_seconds: unix_now_seconds(),
|
||||||
|
detail: outcome.detail,
|
||||||
|
});
|
||||||
|
check
|
||||||
|
.checked_run_dirs
|
||||||
|
.push(record.path.display().to_string());
|
||||||
|
}
|
||||||
|
if check.checked_run_dirs.len() > MAX_CHECKED_RUN_DIRS {
|
||||||
|
let drain = check.checked_run_dirs.len() - MAX_CHECKED_RUN_DIRS;
|
||||||
|
check.checked_run_dirs.drain(..drain);
|
||||||
|
}
|
||||||
|
snapshot.ccr_check = check;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CcrCheckOutcome {
|
||||||
|
outcome: String,
|
||||||
|
duration_seconds: f64,
|
||||||
|
detail: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn execute_ccr_check(bin: &Path, ccr_path: &Path, timeout: Duration) -> CcrCheckOutcome {
|
||||||
|
let started = Instant::now();
|
||||||
|
let log_path = std::env::temp_dir().join(format!(
|
||||||
|
"rpki-artifact-metrics-ccr-check-{}-{}.log",
|
||||||
|
std::process::id(),
|
||||||
|
SystemTime::now()
|
||||||
|
.duration_since(UNIX_EPOCH)
|
||||||
|
.map(|d| d.as_nanos())
|
||||||
|
.unwrap_or(0)
|
||||||
|
));
|
||||||
|
let (outcome, detail) = execute_ccr_check_logged(bin, ccr_path, timeout, &log_path);
|
||||||
|
let _ = fs::remove_file(&log_path);
|
||||||
|
CcrCheckOutcome {
|
||||||
|
outcome,
|
||||||
|
duration_seconds: started.elapsed().as_secs_f64(),
|
||||||
|
detail,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn execute_ccr_check_logged(
|
||||||
|
bin: &Path,
|
||||||
|
ccr_path: &Path,
|
||||||
|
timeout: Duration,
|
||||||
|
log_path: &Path,
|
||||||
|
) -> (String, Option<String>) {
|
||||||
|
let log_file = match fs::File::create(log_path) {
|
||||||
|
Ok(file) => file,
|
||||||
|
Err(err) => {
|
||||||
|
return (
|
||||||
|
"tool_error".to_string(),
|
||||||
|
Some(format!("create ccr check log failed: {err}")),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let log_err = match log_file.try_clone() {
|
||||||
|
Ok(file) => file,
|
||||||
|
Err(err) => {
|
||||||
|
return (
|
||||||
|
"tool_error".to_string(),
|
||||||
|
Some(format!("clone ccr check log handle failed: {err}")),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let mut child = match Command::new(bin)
|
||||||
|
.arg("-f")
|
||||||
|
.arg(ccr_path)
|
||||||
|
.stdin(Stdio::null())
|
||||||
|
.stdout(Stdio::from(log_file))
|
||||||
|
.stderr(Stdio::from(log_err))
|
||||||
|
.spawn()
|
||||||
|
{
|
||||||
|
Ok(child) => child,
|
||||||
|
Err(err) => {
|
||||||
|
return (
|
||||||
|
"tool_error".to_string(),
|
||||||
|
Some(format!("spawn ccr check failed: {err}")),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let started = Instant::now();
|
||||||
|
let exit_code = loop {
|
||||||
|
match child.try_wait() {
|
||||||
|
Ok(Some(status)) => break status.code(),
|
||||||
|
Ok(None) => {
|
||||||
|
if started.elapsed() >= timeout {
|
||||||
|
let _ = child.kill();
|
||||||
|
let _ = child.wait();
|
||||||
|
return (
|
||||||
|
"tool_error".to_string(),
|
||||||
|
Some(format!("ccr check timed out after {}s", timeout.as_secs())),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
thread::sleep(Duration::from_millis(100));
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
return (
|
||||||
|
"tool_error".to_string(),
|
||||||
|
Some(format!("wait ccr check failed: {err}")),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let output = fs::read_to_string(log_path).unwrap_or_default();
|
||||||
|
classify_ccr_check_output(exit_code, ccr_path, &output)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn classify_ccr_check_output(
|
||||||
|
exit_code: Option<i32>,
|
||||||
|
ccr_path: &Path,
|
||||||
|
output: &str,
|
||||||
|
) -> (String, Option<String>) {
|
||||||
|
if exit_code != Some(0) {
|
||||||
|
return (
|
||||||
|
"tool_error".to_string(),
|
||||||
|
Some(format!("rpki-client exited with code {exit_code:?}")),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// rpki-client exits 0 even when the CCR file itself fails to parse, so
|
||||||
|
// the file-level error line is the authoritative failure signal. This
|
||||||
|
// mirrors scripts/ccr/check_rpki_client_ccr.sh.
|
||||||
|
let marker = format!("rpki-client: {}:", ccr_path.display());
|
||||||
|
if let Some(line) = output.lines().find(|line| line.contains(&marker)) {
|
||||||
|
return ("ccr_parse_error".to_string(), Some(line.trim().to_string()));
|
||||||
|
}
|
||||||
|
("pass".to_string(), None)
|
||||||
|
}
|
||||||
|
|
||||||
fn read_json_optional(path: &Path, errors: &mut Vec<String>) -> Option<Value> {
|
fn read_json_optional(path: &Path, errors: &mut Vec<String>) -> Option<Value> {
|
||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
return None;
|
return None;
|
||||||
@ -1634,10 +1945,60 @@ fn render_metrics(snapshot: &MetricsSnapshot) -> String {
|
|||||||
if let Some(ccr) = snapshot.ccr.as_ref() {
|
if let Some(ccr) = snapshot.ccr.as_ref() {
|
||||||
render_ccr_metrics(&mut writer, instance, ccr);
|
render_ccr_metrics(&mut writer, instance, ccr);
|
||||||
}
|
}
|
||||||
|
render_ccr_check_metrics(&mut writer, instance, &snapshot.ccr_check);
|
||||||
render_rtr_metrics(&mut writer, instance, snapshot);
|
render_rtr_metrics(&mut writer, instance, snapshot);
|
||||||
out
|
out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn render_ccr_check_metrics(writer: &mut PromWriter<'_>, instance: &str, check: &CcrCheckMetrics) {
|
||||||
|
writer.gauge(
|
||||||
|
"ours_rp_ccr_format_check_up",
|
||||||
|
"Whether the CCR rpki-client format checker binary is configured and present",
|
||||||
|
&[label("instance", instance)],
|
||||||
|
bool_value(check.up),
|
||||||
|
);
|
||||||
|
for (result, value) in [
|
||||||
|
("pass", check.pass_total),
|
||||||
|
("ccr_parse_error", check.ccr_parse_error_total),
|
||||||
|
("tool_error", check.tool_error_total),
|
||||||
|
] {
|
||||||
|
writer.counter(
|
||||||
|
"ours_rp_ccr_format_check_total",
|
||||||
|
"CCR rpki-client format check results by classification",
|
||||||
|
&[label("instance", instance), label("result", result)],
|
||||||
|
value as f64,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
writer.gauge(
|
||||||
|
"ours_rp_ccr_format_check_consecutive_failures",
|
||||||
|
"Consecutive failed CCR rpki-client format checks",
|
||||||
|
&[label("instance", instance)],
|
||||||
|
check.consecutive_failures as f64,
|
||||||
|
);
|
||||||
|
if let Some(last) = check.last.as_ref() {
|
||||||
|
writer.gauge(
|
||||||
|
"ours_rp_ccr_format_check_last_success",
|
||||||
|
"Whether the latest CCR rpki-client format check passed",
|
||||||
|
&[label("instance", instance)],
|
||||||
|
bool_value(last.outcome == "pass"),
|
||||||
|
);
|
||||||
|
if let Some(run_seq) = last.run_seq {
|
||||||
|
writer.gauge(
|
||||||
|
"ours_rp_ccr_format_check_last_run_sequence",
|
||||||
|
"Run sequence of the latest checked CCR artifact",
|
||||||
|
&[label("instance", instance)],
|
||||||
|
run_seq as f64,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
writer.gauge(
|
||||||
|
"ours_rp_ccr_format_check_last_duration_seconds",
|
||||||
|
"Duration of the latest CCR rpki-client format check",
|
||||||
|
&[label("instance", instance)],
|
||||||
|
last.duration_seconds,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn render_rtr_metrics(writer: &mut PromWriter<'_>, instance: &str, snapshot: &MetricsSnapshot) {
|
fn render_rtr_metrics(writer: &mut PromWriter<'_>, instance: &str, snapshot: &MetricsSnapshot) {
|
||||||
let rtr = &snapshot.rtr;
|
let rtr = &snapshot.rtr;
|
||||||
let scan_now = snapshot.service.last_scan_timestamp_seconds;
|
let scan_now = snapshot.service.last_scan_timestamp_seconds;
|
||||||
@ -2525,6 +2886,7 @@ fn render_status_json(snapshot: &MetricsSnapshot) -> Result<String, String> {
|
|||||||
"latestRun": snapshot.latest_run,
|
"latestRun": snapshot.latest_run,
|
||||||
"cir": snapshot.cir,
|
"cir": snapshot.cir,
|
||||||
"ccr": snapshot.ccr,
|
"ccr": snapshot.ccr,
|
||||||
|
"ccrCheck": snapshot.ccr_check,
|
||||||
"rtr": snapshot.rtr,
|
"rtr": snapshot.rtr,
|
||||||
"topRepositoriesBySyncDuration": snapshot.top_repos_by_sync_duration,
|
"topRepositoriesBySyncDuration": snapshot.top_repos_by_sync_duration,
|
||||||
"topPublicationPointsByObjectCount": snapshot.top_pp_by_object_count,
|
"topPublicationPointsByObjectCount": snapshot.top_pp_by_object_count,
|
||||||
@ -3162,7 +3524,13 @@ mod tests {
|
|||||||
)
|
)
|
||||||
.expect("clients");
|
.expect("clients");
|
||||||
|
|
||||||
let snapshot = scan_run_root_incremental_with_rtr_dir(td.path(), "test", None, Some(&rtr))
|
let snapshot = scan_run_root_incremental_with_rtr_dir(
|
||||||
|
td.path(),
|
||||||
|
"test",
|
||||||
|
None,
|
||||||
|
Some(&rtr),
|
||||||
|
&CcrCheckConfig::disabled(),
|
||||||
|
)
|
||||||
.expect("scan");
|
.expect("scan");
|
||||||
assert!(snapshot.rtr.enabled);
|
assert!(snapshot.rtr.enabled);
|
||||||
assert_eq!(snapshot.rtr.files["source"].seen, 1);
|
assert_eq!(snapshot.rtr.files["source"].seen, 1);
|
||||||
@ -3233,7 +3601,13 @@ mod tests {
|
|||||||
)
|
)
|
||||||
.expect("clients");
|
.expect("clients");
|
||||||
|
|
||||||
let snapshot = scan_run_root_incremental_with_rtr_dir(td.path(), "test", None, Some(&rtr))
|
let snapshot = scan_run_root_incremental_with_rtr_dir(
|
||||||
|
td.path(),
|
||||||
|
"test",
|
||||||
|
None,
|
||||||
|
Some(&rtr),
|
||||||
|
&CcrCheckConfig::disabled(),
|
||||||
|
)
|
||||||
.expect("scan");
|
.expect("scan");
|
||||||
assert!(snapshot.rtr.enabled);
|
assert!(snapshot.rtr.enabled);
|
||||||
assert_eq!(snapshot.rtr.files["source"].seen, 1);
|
assert_eq!(snapshot.rtr.files["source"].seen, 1);
|
||||||
@ -3247,6 +3621,223 @@ mod tests {
|
|||||||
assert!(metrics.contains(r#"ours_rp_rtr_metrics_enabled{instance="test"} 1"#));
|
assert!(metrics.contains(r#"ours_rp_rtr_metrics_enabled{instance="test"} 1"#));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn parse_args_accepts_ccr_check_options() {
|
||||||
|
let args = parse_args(&[
|
||||||
|
"rpki_artifact_metrics".to_string(),
|
||||||
|
"--run-root".to_string(),
|
||||||
|
"root".to_string(),
|
||||||
|
"--ccr-check-bin".to_string(),
|
||||||
|
"/opt/ours-rp/bin/rpki-client".to_string(),
|
||||||
|
"--ccr-check-timeout-secs".to_string(),
|
||||||
|
"60".to_string(),
|
||||||
|
"--ccr-check-max-per-scan".to_string(),
|
||||||
|
"3".to_string(),
|
||||||
|
])
|
||||||
|
.expect("parse");
|
||||||
|
assert_eq!(
|
||||||
|
args.ccr_check_bin.as_deref(),
|
||||||
|
Some(Path::new("/opt/ours-rp/bin/rpki-client"))
|
||||||
|
);
|
||||||
|
assert_eq!(args.ccr_check_timeout_secs, 60);
|
||||||
|
assert_eq!(args.ccr_check_max_per_scan, 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ccr_check_disabled_without_bin() {
|
||||||
|
let td = TempDir::new().expect("tempdir");
|
||||||
|
write_success_run(td.path(), "run_0001", 1);
|
||||||
|
let snapshot = scan_run_root(td.path(), "test").expect("scan");
|
||||||
|
assert!(!snapshot.ccr_check.enabled);
|
||||||
|
assert!(!snapshot.ccr_check.up);
|
||||||
|
assert!(snapshot.ccr_check.last.is_none());
|
||||||
|
let metrics = render_metrics(&snapshot);
|
||||||
|
assert!(metrics.contains(r#"ours_rp_ccr_format_check_up{instance="test"} 0"#));
|
||||||
|
assert!(!metrics.contains("ours_rp_ccr_format_check_last_success"));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ccr_check_pass_records_metrics() {
|
||||||
|
let td = TempDir::new().expect("tempdir");
|
||||||
|
write_success_run(td.path(), "run_0001", 1);
|
||||||
|
let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 0\n");
|
||||||
|
let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30);
|
||||||
|
assert!(snapshot.ccr_check.up);
|
||||||
|
assert_eq!(snapshot.ccr_check.pass_total, 1);
|
||||||
|
assert_eq!(snapshot.ccr_check.consecutive_failures, 0);
|
||||||
|
let last = snapshot.ccr_check.last.as_ref().expect("last result");
|
||||||
|
assert_eq!(last.outcome, "pass");
|
||||||
|
assert_eq!(last.run_seq, Some(1));
|
||||||
|
let metrics = render_metrics(&snapshot);
|
||||||
|
assert!(
|
||||||
|
metrics.contains(r#"ours_rp_ccr_format_check_total{instance="test",result="pass"} 1"#)
|
||||||
|
);
|
||||||
|
assert!(metrics.contains(r#"ours_rp_ccr_format_check_last_success{instance="test"} 1"#));
|
||||||
|
assert!(
|
||||||
|
metrics.contains(r#"ours_rp_ccr_format_check_last_run_sequence{instance="test"} 1"#)
|
||||||
|
);
|
||||||
|
let status = render_status_json(&snapshot).expect("status");
|
||||||
|
assert!(status.contains(r#""ccrCheck""#));
|
||||||
|
assert!(status.contains(r#""passTotal": 1"#));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ccr_check_parse_error_detected_from_output() {
|
||||||
|
let td = TempDir::new().expect("tempdir");
|
||||||
|
write_success_run(td.path(), "run_0001", 1);
|
||||||
|
let bin = write_fake_rpki_client(
|
||||||
|
td.path(),
|
||||||
|
"#!/bin/sh\necho \"rpki-client: $2: unexpected number of locations\" >&2\nexit 0\n",
|
||||||
|
);
|
||||||
|
let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30);
|
||||||
|
assert_eq!(snapshot.ccr_check.ccr_parse_error_total, 1);
|
||||||
|
assert_eq!(snapshot.ccr_check.consecutive_failures, 1);
|
||||||
|
let last = snapshot.ccr_check.last.as_ref().expect("last result");
|
||||||
|
assert_eq!(last.outcome, "ccr_parse_error");
|
||||||
|
assert!(
|
||||||
|
last.detail
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.contains("unexpected number of locations")
|
||||||
|
);
|
||||||
|
let metrics = render_metrics(&snapshot);
|
||||||
|
assert!(metrics.contains(
|
||||||
|
r#"ours_rp_ccr_format_check_total{instance="test",result="ccr_parse_error"} 1"#
|
||||||
|
));
|
||||||
|
assert!(metrics.contains(r#"ours_rp_ccr_format_check_last_success{instance="test"} 0"#));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ccr_check_nonzero_exit_is_tool_error() {
|
||||||
|
let td = TempDir::new().expect("tempdir");
|
||||||
|
write_success_run(td.path(), "run_0001", 1);
|
||||||
|
let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 3\n");
|
||||||
|
let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30);
|
||||||
|
assert_eq!(snapshot.ccr_check.tool_error_total, 1);
|
||||||
|
assert_eq!(
|
||||||
|
snapshot.ccr_check.last.as_ref().expect("last").outcome,
|
||||||
|
"tool_error"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ccr_check_timeout_is_tool_error() {
|
||||||
|
let td = TempDir::new().expect("tempdir");
|
||||||
|
write_success_run(td.path(), "run_0001", 1);
|
||||||
|
let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nsleep 30\n");
|
||||||
|
let snapshot = scan_with_ccr_check(td.path(), None, &bin, 1);
|
||||||
|
let last = snapshot.ccr_check.last.as_ref().expect("last result");
|
||||||
|
assert_eq!(last.outcome, "tool_error");
|
||||||
|
assert!(
|
||||||
|
last.detail
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or_default()
|
||||||
|
.contains("timed out")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ccr_check_startup_only_checks_latest_and_new_runs_once() {
|
||||||
|
let td = TempDir::new().expect("tempdir");
|
||||||
|
write_success_run(td.path(), "run_0001", 1);
|
||||||
|
write_success_run(td.path(), "run_0002", 2);
|
||||||
|
let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 0\n");
|
||||||
|
|
||||||
|
// First scan at service start: only the latest existing run is
|
||||||
|
// checked, older history is marked as seen without executing checks.
|
||||||
|
let first = scan_with_ccr_check(td.path(), None, &bin, 30);
|
||||||
|
assert_eq!(first.ccr_check.pass_total, 1);
|
||||||
|
assert!(
|
||||||
|
first
|
||||||
|
.ccr_check
|
||||||
|
.last
|
||||||
|
.as_ref()
|
||||||
|
.expect("last")
|
||||||
|
.run_dir
|
||||||
|
.contains("run_0002")
|
||||||
|
);
|
||||||
|
|
||||||
|
// Second scan: pre-existing history must not be backfilled.
|
||||||
|
let second = scan_with_ccr_check(td.path(), Some(&first), &bin, 30);
|
||||||
|
assert_eq!(second.ccr_check.pass_total, 1);
|
||||||
|
|
||||||
|
// A newly completed run is checked exactly once.
|
||||||
|
write_success_run(td.path(), "run_0003", 3);
|
||||||
|
let third = scan_with_ccr_check(td.path(), Some(&second), &bin, 30);
|
||||||
|
assert_eq!(third.ccr_check.pass_total, 2);
|
||||||
|
assert_eq!(
|
||||||
|
third.ccr_check.last.as_ref().expect("last").run_seq,
|
||||||
|
Some(3)
|
||||||
|
);
|
||||||
|
let fourth = scan_with_ccr_check(td.path(), Some(&third), &bin, 30);
|
||||||
|
assert_eq!(fourth.ccr_check.pass_total, 2);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ccr_check_missing_bin_marks_not_up() {
|
||||||
|
let td = TempDir::new().expect("tempdir");
|
||||||
|
write_success_run(td.path(), "run_0001", 1);
|
||||||
|
let missing = td.path().join("no-such-rpki-client");
|
||||||
|
let snapshot = scan_with_ccr_check(td.path(), None, &missing, 30);
|
||||||
|
assert!(snapshot.ccr_check.enabled);
|
||||||
|
assert!(!snapshot.ccr_check.up);
|
||||||
|
assert!(snapshot.ccr_check.last.is_none());
|
||||||
|
assert!(
|
||||||
|
snapshot
|
||||||
|
.service
|
||||||
|
.parse_errors
|
||||||
|
.iter()
|
||||||
|
.any(|error| error.contains("ccr check bin is not a file"))
|
||||||
|
);
|
||||||
|
let metrics = render_metrics(&snapshot);
|
||||||
|
assert!(metrics.contains(r#"ours_rp_ccr_format_check_up{instance="test"} 0"#));
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_success_run(root: &Path, name: &str, seq: u64) {
|
||||||
|
let run = root.join("runs").join(name);
|
||||||
|
fs::create_dir_all(&run).expect("create run");
|
||||||
|
fs::write(
|
||||||
|
run.join("run-meta.json"),
|
||||||
|
format!(
|
||||||
|
r#"{{"status":"success","run_index":{seq},"run_id":"{name}","sync_mode":"snapshot","snapshot_reason":"first_run","started_at_rfc3339_utc":"2026-07-21T00:00:00Z","completed_at_rfc3339_utc":"2026-07-21T00:00:10Z"}}"#
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.expect("meta");
|
||||||
|
fs::write(
|
||||||
|
run.join("run-summary.json"),
|
||||||
|
format!(
|
||||||
|
r#"{{"runSeq":{seq},"runId":"{name}","startedAtRfc3339Utc":"2026-07-21T00:00:00Z","finishedAtRfc3339Utc":"2026-07-21T00:00:10Z","wallMs":10000,"status":"success","exitCode":0}}"#
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.expect("summary");
|
||||||
|
fs::write(run.join("result.ccr"), sample_ccr()).expect("ccr");
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_fake_rpki_client(root: &Path, body: &str) -> PathBuf {
|
||||||
|
let path = root.join("fake-rpki-client.sh");
|
||||||
|
fs::write(&path, body).expect("write fake rpki-client");
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::PermissionsExt;
|
||||||
|
fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod");
|
||||||
|
}
|
||||||
|
path
|
||||||
|
}
|
||||||
|
|
||||||
|
fn scan_with_ccr_check(
|
||||||
|
root: &Path,
|
||||||
|
previous: Option<&MetricsSnapshot>,
|
||||||
|
bin: &Path,
|
||||||
|
timeout_secs: u64,
|
||||||
|
) -> MetricsSnapshot {
|
||||||
|
let config = CcrCheckConfig {
|
||||||
|
bin: Some(bin.to_path_buf()),
|
||||||
|
timeout_secs,
|
||||||
|
max_per_scan: 2,
|
||||||
|
};
|
||||||
|
scan_run_root_incremental_with_rtr_dir(root, "test", previous, None, &config).expect("scan")
|
||||||
|
}
|
||||||
|
|
||||||
fn sample_cir() -> Vec<u8> {
|
fn sample_cir() -> Vec<u8> {
|
||||||
let rejected = vec![CirRejectedObject {
|
let rejected = vec![CirRejectedObject {
|
||||||
object_uri: "rsync://repo.example/a/bad.roa".to_string(),
|
object_uri: "rsync://repo.example/a/bad.roa".to_string(),
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user