NASP/rpki
75
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

20260721 metrics 服务新增 CCR rpki-client 格式检查指标与 Grafana 面板（#125）

Browse Source
This commit is contained in:
yuyr 2026-07-21 18:07:05 +08:00
parent 4074323302
commit f86fbd4939
9 changed files with 956 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore
View File

@ -1,6 +1,7 @@
target/
.git/
.gitignore
docker/
perf.*
**/* copy.excalidraw
ui/rpki-explorer/node_modules/

6
deploy/docker-installer/.env.example
View File

@ -88,6 +88,12 @@ RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS=2000
METRICS_INSTANCE=__PACKAGE_ARCH__-installer
METRICS_PORT=9556
METRICS_POLL_SECS=10
# CCR rpki-client format check: the artifact-metrics container runs
# `rpki-client -f result.ccr` once for every newly completed run and exports
# ours_rp_ccr_format_check_* metrics. The binary is packaged inside the
# metrics image; adjust only when deliberately testing another binary.
CCR_CHECK_BIN=/opt/ours-rp/bin/rpki-client
CCR_CHECK_TIMEOUT_SECS=120
# Optional external RTR report directory produced by a separately deployed RTR service.
# Default points to an installer-managed empty fallback directory. To enable real RTR
# metrics, set this to the host-side report directory, for example:

4
deploy/docker-installer/compose/docker-compose.yml
View File

@ -39,6 +39,10 @@ services:
- ${METRICS_POLL_SECS:-10}
- --instance
- ${METRICS_INSTANCE:-package-installer}
- --ccr-check-bin
- ${CCR_CHECK_BIN:-/opt/ours-rp/bin/rpki-client}
- --ccr-check-timeout-secs
- ${CCR_CHECK_TIMEOUT_SECS:-120}
ports:
- "${METRICS_PORT:-9556}:9556"
volumes:

258
deploy/docker-installer/compose/grafana/dashboards/ours-rp-soak-overview.json
View File

@ -850,6 +850,264 @@
],
"title": "Fresh PP / Object Counts by Run",
"type": "timeseries"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none",
"mappings": [
{
"type": "value",
"options": {
"1": {
"text": "PASS",
"color": "green"
},
"0": {
"text": "FAIL",
"color": "red"
}
}
}
]
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 56,
"w": 6,
"h": 4
},
"id": 19,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_ccr_format_check_last_success and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "CCR check",
"refId": "A"
}
],
"title": "Latest CCR Format Check",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none"
},
"overrides": []
},
"gridPos": {
"x": 6,
"y": 56,
"w": 6,
"h": 4
},
"id": 20,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_ccr_format_check_last_run_sequence and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "run seq",
"refId": "A"
}
],
"title": "CCR Checked Run Sequence",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 1,
"unit": "s"
},
"overrides": []
},
"gridPos": {
"x": 12,
"y": 56,
"w": 6,
"h": 4
},
"id": 21,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_ccr_format_check_last_duration_seconds and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "duration",
"refId": "A"
}
],
"title": "CCR Check Duration",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"decimals": 0,
"unit": "none",
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 1
}
]
}
},
"overrides": []
},
"gridPos": {
"x": 18,
"y": 56,
"w": 6,
"h": 4
},
"id": 22,
"options": {
"colorMode": "value",
"graphMode": "area",
"justifyMode": "auto",
"orientation": "auto",
"reduceOptions": {
"calcs": [
"lastNotNull"
],
"fields": "",
"values": false
},
"textMode": "auto",
"wideLayout": true
},
"pluginVersion": "11.3.1",
"targets": [
{
"expr": "max(ours_rp_ccr_format_check_consecutive_failures and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "failures",
"refId": "A"
}
],
"title": "CCR Check Consecutive Failures",
"type": "stat"
},
{
"datasource": {
"type": "prometheus",
"uid": "Prometheus"
},
"fieldConfig": {
"defaults": {
"unit": "none",
"decimals": 0,
"min": 0
},
"overrides": []
},
"gridPos": {
"x": 0,
"y": 60,
"w": 24,
"h": 8
},
"id": 23,
"options": {
"legend": {
"calcs": [
"lastNotNull",
"max"
],
"displayMode": "table",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "multi",
"sort": "none"
}
},
"targets": [
{
"expr": "sum by (result) (ours_rp_ccr_format_check_total)",
"legendFormat": "{{result}} checks",
"refId": "A"
},
{
"expr": "max(ours_rp_ccr_format_check_last_success and on (exported_instance) topk(1, ours_rp_run_sequence))",
"legendFormat": "latest check ok",
"refId": "B"
}
],
"title": "CCR Format Check Results",
"type": "timeseries"
}
],
"refresh": "5s",

6
deploy/docker-installer/docs/README.en.md
View File

@ -125,6 +125,12 @@ If a separately deployed RTR service on the same host continuously writes `rtr-s
The command accepts only an absolute path on the same host, validates the candidate Compose mount, and atomically updates `.env`. It recreates only `artifact-metrics`, Prometheus, and Grafana; it does not restart or recreate `ours-rp-soak`. The `artifact-metrics` container mounts the directory read-only and reads it through `RPKI_METRICS_RTR_REPORT_DIR`, producing `ours_rp_rtr_*` metrics. By default, `RTR_REPORT_DIR` points at an empty fallback directory under the installer data root, so the metrics container also starts cleanly without an RTR service.
## CCR artifact format check metrics
The metrics image ships the official `rpki-client 9.8`. For every newly completed run, `artifact-metrics` runs `rpki-client -f result.ccr` once and exports `ours_rp_ccr_format_check_*` metrics (cumulative pass/ccr_parse_error/tool_error counters, latest check result, check duration, consecutive failures). The Grafana `Ours RP Soak Overview` dashboard shows them in a dedicated panel row. On service start only the latest existing run is checked; older history is not backfilled.
Related environment variables (normally no need to change): `CCR_CHECK_BIN` (defaults to the in-image `/opt/ours-rp/bin/rpki-client`) and `CCR_CHECK_TIMEOUT_SECS` (default 120 seconds).
## Data Directory
Default host directory:

6
deploy/docker-installer/docs/README.zh-CN.md
View File

@ -125,6 +125,12 @@ GRAFANA_ADMIN_PASSWORD=admin
该命令只接受本机绝对路径，会先校验候选 Compose 挂载，再原子更新 `.env`。随后只重建 `artifact-metrics`、Prometheus 和 Grafana，不重启或重建 `ours-rp-soak`。`artifact-metrics` 以只读方式挂载该目录，并通过 `RPKI_METRICS_RTR_REPORT_DIR` 读取 report，输出 `ours_rp_rtr_*` 指标。默认 `RTR_REPORT_DIR` 指向安装包数据目录下的空 fallback 目录，因此未接入 RTR 服务时 metrics 容器也能稳定启动。
## CCR 产物格式检查监控
metrics 镜像内置官方 `rpki-client 9.8`。`artifact-metrics` 会对每个新完成 run 的 `result.ccr` 执行一次 `rpki-client -f` 格式检查，并输出 `ours_rp_ccr_format_check_*` 指标（累计 pass/ccr_parse_error/tool_error、最近检查结果、检查耗时、连续失败数）。Grafana `Ours RP Soak Overview` 底部有对应面板。服务启动时只补查最新的一个历史 run，不回扫全部历史。
相关环境变量（通常无需修改）：`CCR_CHECK_BIN`（默认为镜像内 `/opt/ours-rp/bin/rpki-client`）、`CCR_CHECK_TIMEOUT_SECS`（默认 120 秒）。
## 数据目录
默认宿主机目录：

12
deploy/docker-installer/scripts/common.sh
View File

@ -756,6 +756,18 @@ verify_metrics_image() {
log "verifying metrics image $METRICS_IMAGE on $METRICS_PLATFORM"
verify_image_platform "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics"
verify_image_usage_output "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics" /opt/ours-rp/bin/rpki_artifact_metrics --help
verify_metrics_image_rpki_client
}
verify_metrics_image_rpki_client() {
local output
local status
set +e
output="$(docker run --rm --platform "$METRICS_PLATFORM" "$METRICS_IMAGE" /opt/ours-rp/bin/rpki-client -V 2>&1)"
status=$?
set -e
[[ "$status" == "0" ]] || die "metrics image rpki-client -V failed: image=$METRICS_IMAGE exit=$status output=$output"
grep -q "rpki-client" <<<"$output" || die "metrics image rpki-client -V unexpected output: $output"
}
verify_image_platform() {

43
docker/ours-rp-metrics.Dockerfile
View File

@ -85,27 +85,66 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
mkdir -p /build-out/bin; \
cp "target/$target_triple/release/rpki_artifact_metrics" /build-out/bin/
FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS rpki-client-builder
ARG RPKI_CLIENT_VERSION=9.8
ARG RPKI_CLIENT_SHA256=42920aac5afd0996173fb9f7848691a2c49dd234e4f10478808c1aa475620861
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
curl \
libexpat1-dev \
libssl-dev \
libtls-dev \
pkg-config \
rsync \
zlib1g-dev \
&& rm -rf /var/lib/apt/lists/* \
&& useradd --system --no-create-home --shell /usr/sbin/nologin _rpki-client
RUN set -eux; \
curl -fsSL "https://ftp.openbsd.org/pub/OpenBSD/rpki-client/rpki-client-${RPKI_CLIENT_VERSION}.tar.gz" -o /tmp/rpki-client.tar.gz; \
echo "${RPKI_CLIENT_SHA256} /tmp/rpki-client.tar.gz" | sha256sum -c -; \
tar -xzf /tmp/rpki-client.tar.gz -C /tmp; \
cd "/tmp/rpki-client-${RPKI_CLIENT_VERSION}"; \
./configure --prefix=/opt/rpki-client; \
make -j"$(nproc)"; \
make install; \
/opt/rpki-client/sbin/rpki-client -V
FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
libexpat1 \
libssl3 \
libtls26 \
tzdata \
&& rm -rf /var/lib/apt/lists/*
zlib1g \
&& rm -rf /var/lib/apt/lists/* \
&& useradd --system --no-create-home --shell /usr/sbin/nologin _rpki-client \
&& mkdir -p /opt/rpki-client/var/cache/rpki-client
WORKDIR /opt/ours-rp
COPY --from=builder /build-out/bin/ /opt/ours-rp/bin/
COPY --from=rpki-client-builder /opt/rpki-client/ /opt/rpki-client/
COPY --from=rpki-client-builder /opt/rpki-client/sbin/rpki-client /opt/ours-rp/bin/rpki-client
ARG SOURCE_COMMIT=unknown
ARG SOURCE_DIRTY=unknown
ARG BUILD_TIMESTAMP_UTC=unknown
ARG RPKI_CLIENT_VERSION=9.8
LABEL org.opencontainers.image.title="ours-rp-metrics" \
org.opencontainers.image.description="Ours RP artifact metrics image for multi-arch Docker Compose deployment" \
org.opencontainers.image.revision="${SOURCE_COMMIT}" \
org.opencontainers.image.created="${BUILD_TIMESTAMP_UTC}" \
org.opencontainers.image.source-dirty="${SOURCE_DIRTY}"
org.opencontainers.image.source-dirty="${SOURCE_DIRTY}" \
org.opencontainers.image.rpki-client.version="${RPKI_CLIENT_VERSION}"
RUN chmod +x /opt/ours-rp/bin/* \
&& mkdir -p /var/lib/ours-rp/state /var/lib/ours-rp/runs /var/lib/ours-rp/logs

653
src/tools/rpki_artifact_metrics.rs
View File

@ -3,6 +3,7 @@ use std::fs;
use std::io::{Read, Write};
use std::net::{TcpListener, TcpStream};
use std::path::{Path, PathBuf};
use std::process::{Command, Stdio};
use std::sync::{Arc, RwLock};
use std::thread;
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
@ -26,10 +27,13 @@ struct Args {
once: bool,
out_metrics: Option<PathBuf>,
out_status: Option<PathBuf>,
ccr_check_bin: Option<PathBuf>,
ccr_check_timeout_secs: u64,
ccr_check_max_per_scan: u64,
}
fn usage() -> &'static str {
"Usage: rpki_artifact_metrics --run-root <path> [--listen <addr:port>] [--poll-secs <n>] [--instance <name>] [--once] [--out-metrics <path>] [--out-status <path>]"
"Usage: rpki_artifact_metrics --run-root <path> [--listen <addr:port>] [--poll-secs <n>] [--instance <name>] [--once] [--out-metrics <path>] [--out-status <path>] [--ccr-check-bin <path>] [--ccr-check-timeout-secs <n>] [--ccr-check-max-per-scan <n>]"
}
pub fn main_entry() -> Result<(), String> {
@ -38,8 +42,16 @@ pub fn main_entry() -> Result<(), String> {
fn real_main() -> Result<(), String> {
let args = parse_args(&std::env::args().collect::<Vec<_>>())?;
let ccr_check_config = CcrCheckConfig::from_args(&args);
let rtr_report_dir = configured_rtr_report_dir();
if args.once {
let snapshot = scan_run_root(&args.run_root, &args.instance)?;
let snapshot = scan_run_root_incremental_with_rtr_dir(
&args.run_root,
&args.instance,
None,
rtr_report_dir.as_deref(),
&ccr_check_config,
)?;
let metrics = render_metrics(&snapshot);
let status = render_status_json(&snapshot)?;
if let Some(path) = args.out_metrics.as_ref() {
@ -53,7 +65,13 @@ fn real_main() -> Result<(), String> {
return Ok(());
}
let shared = Arc::new(RwLock::new(scan_run_root(&args.run_root, &args.instance)?));
let shared = Arc::new(RwLock::new(scan_run_root_incremental_with_rtr_dir(
&args.run_root,
&args.instance,
None,
rtr_report_dir.as_deref(),
&ccr_check_config,
)?));
let scanner = Arc::clone(&shared);
let run_root = args.run_root.clone();
let instance = args.instance.clone();
@ -62,20 +80,25 @@ fn real_main() -> Result<(), String> {
loop {
thread::sleep(Duration::from_secs(poll_secs));
let previous_snapshot = scanner.read().expect("metrics lock poisoned").clone();
let next =
match scan_run_root_incremental(&run_root, &instance, Some(&previous_snapshot)) {
Ok(snapshot) => snapshot,
Err(err) => {
let mut previous = scanner.write().expect("metrics lock poisoned");
previous
.service
.parse_errors
.push(format!("scan failed: {err}"));
previous.service.last_reload_success = false;
previous.service.last_scan_timestamp_seconds = unix_now_seconds();
continue;
}
};
let next = match scan_run_root_incremental_with_rtr_dir(
&run_root,
&instance,
Some(&previous_snapshot),
rtr_report_dir.as_deref(),
&ccr_check_config,
) {
Ok(snapshot) => snapshot,
Err(err) => {
let mut previous = scanner.write().expect("metrics lock poisoned");
previous
.service
.parse_errors
.push(format!("scan failed: {err}"));
previous.service.last_reload_success = false;
previous.service.last_scan_timestamp_seconds = unix_now_seconds();
continue;
}
};
*scanner.write().expect("metrics lock poisoned") = next;
}
});
@ -91,6 +114,9 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
let mut once = false;
let mut out_metrics = None;
let mut out_status = None;
let mut ccr_check_bin = None;
let mut ccr_check_timeout_secs = 120u64;
let mut ccr_check_max_per_scan = 2u64;
let mut index = 1usize;
while index < argv.len() {
match argv[index].as_str() {
@ -122,6 +148,24 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
index += 1;
out_status = Some(PathBuf::from(value_at(argv, index, "--out-status")?));
}
"--ccr-check-bin" => {
index += 1;
ccr_check_bin = Some(PathBuf::from(value_at(argv, index, "--ccr-check-bin")?));
}
"--ccr-check-timeout-secs" => {
index += 1;
let value = value_at(argv, index, "--ccr-check-timeout-secs")?;
ccr_check_timeout_secs = value
.parse::<u64>()
.map_err(|_| format!("invalid --ccr-check-timeout-secs: {value}"))?;
}
"--ccr-check-max-per-scan" => {
index += 1;
let value = value_at(argv, index, "--ccr-check-max-per-scan")?;
ccr_check_max_per_scan = value
.parse::<u64>()
.map_err(|_| format!("invalid --ccr-check-max-per-scan: {value}"))?;
}
"-h" | "--help" => return Err(usage().to_string()),
other => return Err(format!("unknown argument: {other}\n{}", usage())),
}
@ -135,6 +179,9 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
once,
out_metrics,
out_status,
ccr_check_bin,
ccr_check_timeout_secs,
ccr_check_max_per_scan,
})
}
@ -161,9 +208,67 @@ struct MetricsSnapshot {
top_pp_by_sync_duration: Vec<TopPublicationPoint>,
cir: Option<CirMetrics>,
ccr: Option<CcrMetrics>,
ccr_check: CcrCheckMetrics,
rtr: RtrMetrics,
}
#[derive(Clone, Debug, Default, Serialize)]
#[serde(rename_all = "camelCase")]
struct CcrCheckMetrics {
enabled: bool,
up: bool,
bin: Option<String>,
pass_total: u64,
ccr_parse_error_total: u64,
tool_error_total: u64,
consecutive_failures: u64,
last: Option<CcrCheckLastResult>,
checked_run_dirs: Vec<String>,
}
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "camelCase")]
struct CcrCheckLastResult {
run_dir: String,
run_seq: Option<u64>,
outcome: String,
duration_seconds: f64,
checked_at_seconds: f64,
detail: Option<String>,
}
#[derive(Clone, Debug)]
struct CcrCheckConfig {
bin: Option<PathBuf>,
timeout_secs: u64,
max_per_scan: u64,
}
impl CcrCheckConfig {
fn disabled() -> Self {
Self {
bin: None,
timeout_secs: 120,
max_per_scan: 2,
}
}
fn from_args(args: &Args) -> Self {
let bin = args.ccr_check_bin.clone().or_else(|| {
std::env::var("RPKI_METRICS_CCR_CHECK_BIN")
.ok()
.map(|value| value.trim().to_string())
.filter(|value| !value.is_empty())
.map(PathBuf::from)
});
Self {
bin,
timeout_secs: args.ccr_check_timeout_secs,
max_per_scan: args.ccr_check_max_per_scan,
}
}
}
#[derive(Clone, Debug, Default, Serialize)]
#[serde(rename_all = "camelCase")]
struct ServiceMetrics {
@ -493,21 +598,13 @@ struct RunRecord {
}
fn scan_run_root(input_root: &Path, instance: &str) -> Result<MetricsSnapshot, String> {
let rtr_report_dir = configured_rtr_report_dir();
scan_run_root_incremental_with_rtr_dir(input_root, instance, None, rtr_report_dir.as_deref())
}
fn scan_run_root_incremental(
input_root: &Path,
instance: &str,
previous: Option<&MetricsSnapshot>,
) -> Result<MetricsSnapshot, String> {
let rtr_report_dir = configured_rtr_report_dir();
scan_run_root_incremental_with_rtr_dir(
input_root,
instance,
previous,
None,
rtr_report_dir.as_deref(),
&CcrCheckConfig::disabled(),
)
}
@ -516,6 +613,7 @@ fn scan_run_root_incremental_with_rtr_dir(
instance: &str,
previous: Option<&MetricsSnapshot>,
rtr_report_dir: Option<&Path>,
ccr_check_config: &CcrCheckConfig,
) -> Result<MetricsSnapshot, String> {
let started = Instant::now();
let runs_root = resolve_runs_root(input_root);
@ -570,6 +668,7 @@ fn scan_run_root_incremental_with_rtr_dir(
}
snapshot.rtr = scan_rtr_reports(rtr_report_dir, previous.map(|snapshot| &snapshot.rtr));
run_pending_ccr_checks(&records, previous, ccr_check_config, &mut snapshot);
snapshot.service.last_scan_timestamp_seconds = unix_now_seconds();
snapshot.service.last_scan_duration_seconds = started.elapsed().as_secs_f64();
snapshot.service.last_reload_success =
@ -975,6 +1074,218 @@ fn consecutive_failures(records: &[RunRecord]) -> u64 {
count
}
const MAX_CHECKED_RUN_DIRS: usize = 512;
fn run_pending_ccr_checks(
records: &[RunRecord],
previous: Option<&MetricsSnapshot>,
config: &CcrCheckConfig,
snapshot: &mut MetricsSnapshot,
) {
let mut check = previous
.map(|prev| prev.ccr_check.clone())
.unwrap_or_default();
check.enabled = config.bin.is_some();
check.bin = config.bin.as_ref().map(|bin| bin.display().to_string());
check.up = false;
let Some(bin) = config.bin.as_ref() else {
snapshot.ccr_check = check;
return;
};
if !bin.is_file() {
snapshot
.service
.parse_errors
.push(format!("ccr check bin is not a file: {}", bin.display()));
snapshot.ccr_check = check;
return;
}
check.up = true;
let mut pending: Vec<&RunRecord> = records
.iter()
.filter(|record| record.status == "success")
.filter(|record| record.path.join("result.ccr").is_file())
.filter(|record| {
let dir = record.path.display().to_string();
!check.checked_run_dirs.iter().any(|seen| seen == &dir)
})
.collect();
if previous.is_none() {
// On service start only the latest existing run is checked; older
// history is marked as seen without executing checks.
let latest_path = pending.last().map(|record| record.path.clone());
for record in &pending {
if Some(&record.path) != latest_path.as_ref() {
check
.checked_run_dirs
.push(record.path.display().to_string());
}
}
pending.retain(|record| Some(&record.path) == latest_path.as_ref());
}
for record in pending
.into_iter()
.take(config.max_per_scan.max(1) as usize)
{
let ccr_path = record.path.join("result.ccr");
let outcome = execute_ccr_check(bin, &ccr_path, Duration::from_secs(config.timeout_secs));
let run_seq = record
.summary
.as_ref()
.and_then(|v| json_u64(v, &["runSeq"]))
.or_else(|| {
record
.meta
.as_ref()
.and_then(|v| json_u64(v, &["run_index"]))
});
let passed = outcome.outcome == "pass";
match outcome.outcome.as_str() {
"pass" => check.pass_total += 1,
"ccr_parse_error" => check.ccr_parse_error_total += 1,
_ => check.tool_error_total += 1,
}
check.consecutive_failures = if passed {
0
} else {
check.consecutive_failures + 1
};
check.last = Some(CcrCheckLastResult {
run_dir: record.path.display().to_string(),
run_seq,
outcome: outcome.outcome,
duration_seconds: outcome.duration_seconds,
checked_at_seconds: unix_now_seconds(),
detail: outcome.detail,
});
check
.checked_run_dirs
.push(record.path.display().to_string());
}
if check.checked_run_dirs.len() > MAX_CHECKED_RUN_DIRS {
let drain = check.checked_run_dirs.len() - MAX_CHECKED_RUN_DIRS;
check.checked_run_dirs.drain(..drain);
}
snapshot.ccr_check = check;
}
struct CcrCheckOutcome {
outcome: String,
duration_seconds: f64,
detail: Option<String>,
}
fn execute_ccr_check(bin: &Path, ccr_path: &Path, timeout: Duration) -> CcrCheckOutcome {
let started = Instant::now();
let log_path = std::env::temp_dir().join(format!(
"rpki-artifact-metrics-ccr-check-{}-{}.log",
std::process::id(),
SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0)
));
let (outcome, detail) = execute_ccr_check_logged(bin, ccr_path, timeout, &log_path);
let _ = fs::remove_file(&log_path);
CcrCheckOutcome {
outcome,
duration_seconds: started.elapsed().as_secs_f64(),
detail,
}
}
fn execute_ccr_check_logged(
bin: &Path,
ccr_path: &Path,
timeout: Duration,
log_path: &Path,
) -> (String, Option<String>) {
let log_file = match fs::File::create(log_path) {
Ok(file) => file,
Err(err) => {
return (
"tool_error".to_string(),
Some(format!("create ccr check log failed: {err}")),
);
}
};
let log_err = match log_file.try_clone() {
Ok(file) => file,
Err(err) => {
return (
"tool_error".to_string(),
Some(format!("clone ccr check log handle failed: {err}")),
);
}
};
let mut child = match Command::new(bin)
.arg("-f")
.arg(ccr_path)
.stdin(Stdio::null())
.stdout(Stdio::from(log_file))
.stderr(Stdio::from(log_err))
.spawn()
{
Ok(child) => child,
Err(err) => {
return (
"tool_error".to_string(),
Some(format!("spawn ccr check failed: {err}")),
);
}
};
let started = Instant::now();
let exit_code = loop {
match child.try_wait() {
Ok(Some(status)) => break status.code(),
Ok(None) => {
if started.elapsed() >= timeout {
let _ = child.kill();
let _ = child.wait();
return (
"tool_error".to_string(),
Some(format!("ccr check timed out after {}s", timeout.as_secs())),
);
}
thread::sleep(Duration::from_millis(100));
}
Err(err) => {
return (
"tool_error".to_string(),
Some(format!("wait ccr check failed: {err}")),
);
}
}
};
let output = fs::read_to_string(log_path).unwrap_or_default();
classify_ccr_check_output(exit_code, ccr_path, &output)
}
fn classify_ccr_check_output(
exit_code: Option<i32>,
ccr_path: &Path,
output: &str,
) -> (String, Option<String>) {
if exit_code != Some(0) {
return (
"tool_error".to_string(),
Some(format!("rpki-client exited with code {exit_code:?}")),
);
}
// rpki-client exits 0 even when the CCR file itself fails to parse, so
// the file-level error line is the authoritative failure signal. This
// mirrors scripts/ccr/check_rpki_client_ccr.sh.
let marker = format!("rpki-client: {}:", ccr_path.display());
if let Some(line) = output.lines().find(|line| line.contains(&marker)) {
return ("ccr_parse_error".to_string(), Some(line.trim().to_string()));
}
("pass".to_string(), None)
}
fn read_json_optional(path: &Path, errors: &mut Vec<String>) -> Option<Value> {
if !path.exists() {
return None;
@ -1634,10 +1945,60 @@ fn render_metrics(snapshot: &MetricsSnapshot) -> String {
if let Some(ccr) = snapshot.ccr.as_ref() {
render_ccr_metrics(&mut writer, instance, ccr);
}
render_ccr_check_metrics(&mut writer, instance, &snapshot.ccr_check);
render_rtr_metrics(&mut writer, instance, snapshot);
out
}
fn render_ccr_check_metrics(writer: &mut PromWriter<'_>, instance: &str, check: &CcrCheckMetrics) {
writer.gauge(
"ours_rp_ccr_format_check_up",
"Whether the CCR rpki-client format checker binary is configured and present",
&[label("instance", instance)],
bool_value(check.up),
);
for (result, value) in [
("pass", check.pass_total),
("ccr_parse_error", check.ccr_parse_error_total),
("tool_error", check.tool_error_total),
] {
writer.counter(
"ours_rp_ccr_format_check_total",
"CCR rpki-client format check results by classification",
&[label("instance", instance), label("result", result)],
value as f64,
);
}
writer.gauge(
"ours_rp_ccr_format_check_consecutive_failures",
"Consecutive failed CCR rpki-client format checks",
&[label("instance", instance)],
check.consecutive_failures as f64,
);
if let Some(last) = check.last.as_ref() {
writer.gauge(
"ours_rp_ccr_format_check_last_success",
"Whether the latest CCR rpki-client format check passed",
&[label("instance", instance)],
bool_value(last.outcome == "pass"),
);
if let Some(run_seq) = last.run_seq {
writer.gauge(
"ours_rp_ccr_format_check_last_run_sequence",
"Run sequence of the latest checked CCR artifact",
&[label("instance", instance)],
run_seq as f64,
);
}
writer.gauge(
"ours_rp_ccr_format_check_last_duration_seconds",
"Duration of the latest CCR rpki-client format check",
&[label("instance", instance)],
last.duration_seconds,
);
}
}
fn render_rtr_metrics(writer: &mut PromWriter<'_>, instance: &str, snapshot: &MetricsSnapshot) {
let rtr = &snapshot.rtr;
let scan_now = snapshot.service.last_scan_timestamp_seconds;
@ -2525,6 +2886,7 @@ fn render_status_json(snapshot: &MetricsSnapshot) -> Result<String, String> {
"latestRun": snapshot.latest_run,
"cir": snapshot.cir,
"ccr": snapshot.ccr,
"ccrCheck": snapshot.ccr_check,
"rtr": snapshot.rtr,
"topRepositoriesBySyncDuration": snapshot.top_repos_by_sync_duration,
"topPublicationPointsByObjectCount": snapshot.top_pp_by_object_count,
@ -3162,8 +3524,14 @@ mod tests {
)
.expect("clients");
let snapshot = scan_run_root_incremental_with_rtr_dir(td.path(), "test", None, Some(&rtr))
.expect("scan");
let snapshot = scan_run_root_incremental_with_rtr_dir(
td.path(),
"test",
None,
Some(&rtr),
&CcrCheckConfig::disabled(),
)
.expect("scan");
assert!(snapshot.rtr.enabled);
assert_eq!(snapshot.rtr.files["source"].seen, 1);
assert_eq!(
@ -3233,8 +3601,14 @@ mod tests {
)
.expect("clients");
let snapshot = scan_run_root_incremental_with_rtr_dir(td.path(), "test", None, Some(&rtr))
.expect("scan");
let snapshot = scan_run_root_incremental_with_rtr_dir(
td.path(),
"test",
None,
Some(&rtr),
&CcrCheckConfig::disabled(),
)
.expect("scan");
assert!(snapshot.rtr.enabled);
assert_eq!(snapshot.rtr.files["source"].seen, 1);
assert!(snapshot.rtr.source.is_none());
@ -3247,6 +3621,223 @@ mod tests {
assert!(metrics.contains(r#"ours_rp_rtr_metrics_enabled{instance="test"} 1"#));
}
#[test]
fn parse_args_accepts_ccr_check_options() {
let args = parse_args(&[
"rpki_artifact_metrics".to_string(),
"--run-root".to_string(),
"root".to_string(),
"--ccr-check-bin".to_string(),
"/opt/ours-rp/bin/rpki-client".to_string(),
"--ccr-check-timeout-secs".to_string(),
"60".to_string(),
"--ccr-check-max-per-scan".to_string(),
"3".to_string(),
])
.expect("parse");
assert_eq!(
args.ccr_check_bin.as_deref(),
Some(Path::new("/opt/ours-rp/bin/rpki-client"))
);
assert_eq!(args.ccr_check_timeout_secs, 60);
assert_eq!(args.ccr_check_max_per_scan, 3);
}
#[test]
fn ccr_check_disabled_without_bin() {
let td = TempDir::new().expect("tempdir");
write_success_run(td.path(), "run_0001", 1);
let snapshot = scan_run_root(td.path(), "test").expect("scan");
assert!(!snapshot.ccr_check.enabled);
assert!(!snapshot.ccr_check.up);
assert!(snapshot.ccr_check.last.is_none());
let metrics = render_metrics(&snapshot);
assert!(metrics.contains(r#"ours_rp_ccr_format_check_up{instance="test"} 0"#));
assert!(!metrics.contains("ours_rp_ccr_format_check_last_success"));
}
#[test]
fn ccr_check_pass_records_metrics() {
let td = TempDir::new().expect("tempdir");
write_success_run(td.path(), "run_0001", 1);
let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 0\n");
let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30);
assert!(snapshot.ccr_check.up);
assert_eq!(snapshot.ccr_check.pass_total, 1);
assert_eq!(snapshot.ccr_check.consecutive_failures, 0);
let last = snapshot.ccr_check.last.as_ref().expect("last result");
assert_eq!(last.outcome, "pass");
assert_eq!(last.run_seq, Some(1));
let metrics = render_metrics(&snapshot);
assert!(
metrics.contains(r#"ours_rp_ccr_format_check_total{instance="test",result="pass"} 1"#)
);
assert!(metrics.contains(r#"ours_rp_ccr_format_check_last_success{instance="test"} 1"#));
assert!(
metrics.contains(r#"ours_rp_ccr_format_check_last_run_sequence{instance="test"} 1"#)
);
let status = render_status_json(&snapshot).expect("status");
assert!(status.contains(r#""ccrCheck""#));
assert!(status.contains(r#""passTotal": 1"#));
}
#[test]
fn ccr_check_parse_error_detected_from_output() {
let td = TempDir::new().expect("tempdir");
write_success_run(td.path(), "run_0001", 1);
let bin = write_fake_rpki_client(
td.path(),
"#!/bin/sh\necho \"rpki-client: $2: unexpected number of locations\" >&2\nexit 0\n",
);
let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30);
assert_eq!(snapshot.ccr_check.ccr_parse_error_total, 1);
assert_eq!(snapshot.ccr_check.consecutive_failures, 1);
let last = snapshot.ccr_check.last.as_ref().expect("last result");
assert_eq!(last.outcome, "ccr_parse_error");
assert!(
last.detail
.as_deref()
.unwrap_or_default()
.contains("unexpected number of locations")
);
let metrics = render_metrics(&snapshot);
assert!(metrics.contains(
r#"ours_rp_ccr_format_check_total{instance="test",result="ccr_parse_error"} 1"#
));
assert!(metrics.contains(r#"ours_rp_ccr_format_check_last_success{instance="test"} 0"#));
}
#[test]
fn ccr_check_nonzero_exit_is_tool_error() {
let td = TempDir::new().expect("tempdir");
write_success_run(td.path(), "run_0001", 1);
let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 3\n");
let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30);
assert_eq!(snapshot.ccr_check.tool_error_total, 1);
assert_eq!(
snapshot.ccr_check.last.as_ref().expect("last").outcome,
"tool_error"
);
}
#[test]
fn ccr_check_timeout_is_tool_error() {
let td = TempDir::new().expect("tempdir");
write_success_run(td.path(), "run_0001", 1);
let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nsleep 30\n");
let snapshot = scan_with_ccr_check(td.path(), None, &bin, 1);
let last = snapshot.ccr_check.last.as_ref().expect("last result");
assert_eq!(last.outcome, "tool_error");
assert!(
last.detail
.as_deref()
.unwrap_or_default()
.contains("timed out")
);
}
#[test]
fn ccr_check_startup_only_checks_latest_and_new_runs_once() {
let td = TempDir::new().expect("tempdir");
write_success_run(td.path(), "run_0001", 1);
write_success_run(td.path(), "run_0002", 2);
let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 0\n");
// First scan at service start: only the latest existing run is
// checked, older history is marked as seen without executing checks.
let first = scan_with_ccr_check(td.path(), None, &bin, 30);
assert_eq!(first.ccr_check.pass_total, 1);
assert!(
first
.ccr_check
.last
.as_ref()
.expect("last")
.run_dir
.contains("run_0002")
);
// Second scan: pre-existing history must not be backfilled.
let second = scan_with_ccr_check(td.path(), Some(&first), &bin, 30);
assert_eq!(second.ccr_check.pass_total, 1);
// A newly completed run is checked exactly once.
write_success_run(td.path(), "run_0003", 3);
let third = scan_with_ccr_check(td.path(), Some(&second), &bin, 30);
assert_eq!(third.ccr_check.pass_total, 2);
assert_eq!(
third.ccr_check.last.as_ref().expect("last").run_seq,
Some(3)
);
let fourth = scan_with_ccr_check(td.path(), Some(&third), &bin, 30);
assert_eq!(fourth.ccr_check.pass_total, 2);
}
#[test]
fn ccr_check_missing_bin_marks_not_up() {
let td = TempDir::new().expect("tempdir");
write_success_run(td.path(), "run_0001", 1);
let missing = td.path().join("no-such-rpki-client");
let snapshot = scan_with_ccr_check(td.path(), None, &missing, 30);
assert!(snapshot.ccr_check.enabled);
assert!(!snapshot.ccr_check.up);
assert!(snapshot.ccr_check.last.is_none());
assert!(
snapshot
.service
.parse_errors
.iter()
.any(|error| error.contains("ccr check bin is not a file"))
);
let metrics = render_metrics(&snapshot);
assert!(metrics.contains(r#"ours_rp_ccr_format_check_up{instance="test"} 0"#));
}
fn write_success_run(root: &Path, name: &str, seq: u64) {
let run = root.join("runs").join(name);
fs::create_dir_all(&run).expect("create run");
fs::write(
run.join("run-meta.json"),
format!(
r#"{{"status":"success","run_index":{seq},"run_id":"{name}","sync_mode":"snapshot","snapshot_reason":"first_run","started_at_rfc3339_utc":"2026-07-21T00:00:00Z","completed_at_rfc3339_utc":"2026-07-21T00:00:10Z"}}"#
),
)
.expect("meta");
fs::write(
run.join("run-summary.json"),
format!(
r#"{{"runSeq":{seq},"runId":"{name}","startedAtRfc3339Utc":"2026-07-21T00:00:00Z","finishedAtRfc3339Utc":"2026-07-21T00:00:10Z","wallMs":10000,"status":"success","exitCode":0}}"#
),
)
.expect("summary");
fs::write(run.join("result.ccr"), sample_ccr()).expect("ccr");
}
fn write_fake_rpki_client(root: &Path, body: &str) -> PathBuf {
let path = root.join("fake-rpki-client.sh");
fs::write(&path, body).expect("write fake rpki-client");
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod");
}
path
}
fn scan_with_ccr_check(
root: &Path,
previous: Option<&MetricsSnapshot>,
bin: &Path,
timeout_secs: u64,
) -> MetricsSnapshot {
let config = CcrCheckConfig {
bin: Some(bin.to_path_buf()),
timeout_secs,
max_per_scan: 2,
};
scan_run_root_incremental_with_rtr_dir(root, "test", previous, None, &config).expect("scan")
}
fn sample_cir() -> Vec<u8> {
let rejected = vec![CirRejectedObject {
object_uri: "rsync://repo.example/a/bad.roa".to_string(),