diff --git a/.dockerignore b/.dockerignore index 193ed56..aafd1e7 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,6 +1,7 @@ target/ .git/ .gitignore +docker/ perf.* **/* copy.excalidraw ui/rpki-explorer/node_modules/ diff --git a/deploy/docker-installer/.env.example b/deploy/docker-installer/.env.example index 932f0ef..c0ba16b 100644 --- a/deploy/docker-installer/.env.example +++ b/deploy/docker-installer/.env.example @@ -88,6 +88,12 @@ RPKI_PROGRESS_CONTROL_LOOP_SLOW_MS=2000 METRICS_INSTANCE=__PACKAGE_ARCH__-installer METRICS_PORT=9556 METRICS_POLL_SECS=10 +# CCR rpki-client format check: the artifact-metrics container runs +# `rpki-client -f result.ccr` once for every newly completed run and exports +# ours_rp_ccr_format_check_* metrics. The binary is packaged inside the +# metrics image; adjust only when deliberately testing another binary. +CCR_CHECK_BIN=/opt/ours-rp/bin/rpki-client +CCR_CHECK_TIMEOUT_SECS=120 # Optional external RTR report directory produced by a separately deployed RTR service. # Default points to an installer-managed empty fallback directory. To enable real RTR # metrics, set this to the host-side report directory, for example: diff --git a/deploy/docker-installer/compose/docker-compose.yml b/deploy/docker-installer/compose/docker-compose.yml index 85e660d..43c1110 100644 --- a/deploy/docker-installer/compose/docker-compose.yml +++ b/deploy/docker-installer/compose/docker-compose.yml @@ -39,6 +39,10 @@ services: - ${METRICS_POLL_SECS:-10} - --instance - ${METRICS_INSTANCE:-package-installer} + - --ccr-check-bin + - ${CCR_CHECK_BIN:-/opt/ours-rp/bin/rpki-client} + - --ccr-check-timeout-secs + - ${CCR_CHECK_TIMEOUT_SECS:-120} ports: - "${METRICS_PORT:-9556}:9556" volumes: diff --git a/deploy/docker-installer/compose/grafana/dashboards/ours-rp-soak-overview.json b/deploy/docker-installer/compose/grafana/dashboards/ours-rp-soak-overview.json index a235b53..8f47bb3 100644 --- a/deploy/docker-installer/compose/grafana/dashboards/ours-rp-soak-overview.json +++ b/deploy/docker-installer/compose/grafana/dashboards/ours-rp-soak-overview.json @@ -850,6 +850,264 @@ ], "title": "Fresh PP / Object Counts by Run", "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "none", + "mappings": [ + { + "type": "value", + "options": { + "1": { + "text": "PASS", + "color": "green" + }, + "0": { + "text": "FAIL", + "color": "red" + } + } + } + ] + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 56, + "w": 6, + "h": 4 + }, + "id": 19, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "expr": "max(ours_rp_ccr_format_check_last_success and on (exported_instance) topk(1, ours_rp_run_sequence))", + "legendFormat": "CCR check", + "refId": "A" + } + ], + "title": "Latest CCR Format Check", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "x": 6, + "y": 56, + "w": 6, + "h": 4 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "expr": "max(ours_rp_ccr_format_check_last_run_sequence and on (exported_instance) topk(1, ours_rp_run_sequence))", + "legendFormat": "run seq", + "refId": "A" + } + ], + "title": "CCR Checked Run Sequence", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 1, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "x": 12, + "y": 56, + "w": 6, + "h": 4 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "expr": "max(ours_rp_ccr_format_check_last_duration_seconds and on (exported_instance) topk(1, ours_rp_run_sequence))", + "legendFormat": "duration", + "refId": "A" + } + ], + "title": "CCR Check Duration", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "decimals": 0, + "unit": "none", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "x": 18, + "y": 56, + "w": 6, + "h": 4 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.3.1", + "targets": [ + { + "expr": "max(ours_rp_ccr_format_check_consecutive_failures and on (exported_instance) topk(1, ours_rp_run_sequence))", + "legendFormat": "failures", + "refId": "A" + } + ], + "title": "CCR Check Consecutive Failures", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "unit": "none", + "decimals": 0, + "min": 0 + }, + "overrides": [] + }, + "gridPos": { + "x": 0, + "y": 60, + "w": 24, + "h": 8 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "targets": [ + { + "expr": "sum by (result) (ours_rp_ccr_format_check_total)", + "legendFormat": "{{result}} checks", + "refId": "A" + }, + { + "expr": "max(ours_rp_ccr_format_check_last_success and on (exported_instance) topk(1, ours_rp_run_sequence))", + "legendFormat": "latest check ok", + "refId": "B" + } + ], + "title": "CCR Format Check Results", + "type": "timeseries" } ], "refresh": "5s", diff --git a/deploy/docker-installer/docs/README.en.md b/deploy/docker-installer/docs/README.en.md index 0f11509..21d72a8 100644 --- a/deploy/docker-installer/docs/README.en.md +++ b/deploy/docker-installer/docs/README.en.md @@ -125,6 +125,12 @@ If a separately deployed RTR service on the same host continuously writes `rtr-s The command accepts only an absolute path on the same host, validates the candidate Compose mount, and atomically updates `.env`. It recreates only `artifact-metrics`, Prometheus, and Grafana; it does not restart or recreate `ours-rp-soak`. The `artifact-metrics` container mounts the directory read-only and reads it through `RPKI_METRICS_RTR_REPORT_DIR`, producing `ours_rp_rtr_*` metrics. By default, `RTR_REPORT_DIR` points at an empty fallback directory under the installer data root, so the metrics container also starts cleanly without an RTR service. +## CCR artifact format check metrics + +The metrics image ships the official `rpki-client 9.8`. For every newly completed run, `artifact-metrics` runs `rpki-client -f result.ccr` once and exports `ours_rp_ccr_format_check_*` metrics (cumulative pass/ccr_parse_error/tool_error counters, latest check result, check duration, consecutive failures). The Grafana `Ours RP Soak Overview` dashboard shows them in a dedicated panel row. On service start only the latest existing run is checked; older history is not backfilled. + +Related environment variables (normally no need to change): `CCR_CHECK_BIN` (defaults to the in-image `/opt/ours-rp/bin/rpki-client`) and `CCR_CHECK_TIMEOUT_SECS` (default 120 seconds). + ## Data Directory Default host directory: diff --git a/deploy/docker-installer/docs/README.zh-CN.md b/deploy/docker-installer/docs/README.zh-CN.md index b627e40..78d05e4 100644 --- a/deploy/docker-installer/docs/README.zh-CN.md +++ b/deploy/docker-installer/docs/README.zh-CN.md @@ -125,6 +125,12 @@ GRAFANA_ADMIN_PASSWORD=admin 该命令只接受本机绝对路径，会先校验候选 Compose 挂载，再原子更新 `.env`。随后只重建 `artifact-metrics`、Prometheus 和 Grafana，不重启或重建 `ours-rp-soak`。`artifact-metrics` 以只读方式挂载该目录，并通过 `RPKI_METRICS_RTR_REPORT_DIR` 读取 report，输出 `ours_rp_rtr_*` 指标。默认 `RTR_REPORT_DIR` 指向安装包数据目录下的空 fallback 目录，因此未接入 RTR 服务时 metrics 容器也能稳定启动。 +## CCR 产物格式检查监控 + +metrics 镜像内置官方 `rpki-client 9.8`。`artifact-metrics` 会对每个新完成 run 的 `result.ccr` 执行一次 `rpki-client -f` 格式检查，并输出 `ours_rp_ccr_format_check_*` 指标（累计 pass/ccr_parse_error/tool_error、最近检查结果、检查耗时、连续失败数）。Grafana `Ours RP Soak Overview` 底部有对应面板。服务启动时只补查最新的一个历史 run，不回扫全部历史。 + +相关环境变量（通常无需修改）：`CCR_CHECK_BIN`（默认为镜像内 `/opt/ours-rp/bin/rpki-client`）、`CCR_CHECK_TIMEOUT_SECS`（默认 120 秒）。 + ## 数据目录 默认宿主机目录： diff --git a/deploy/docker-installer/scripts/common.sh b/deploy/docker-installer/scripts/common.sh index 163f28e..978911a 100644 --- a/deploy/docker-installer/scripts/common.sh +++ b/deploy/docker-installer/scripts/common.sh @@ -756,6 +756,18 @@ verify_metrics_image() { log "verifying metrics image $METRICS_IMAGE on $METRICS_PLATFORM" verify_image_platform "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics" verify_image_usage_output "$METRICS_IMAGE" "$METRICS_PLATFORM" "metrics" /opt/ours-rp/bin/rpki_artifact_metrics --help + verify_metrics_image_rpki_client +} + +verify_metrics_image_rpki_client() { + local output + local status + set +e + output="$(docker run --rm --platform "$METRICS_PLATFORM" "$METRICS_IMAGE" /opt/ours-rp/bin/rpki-client -V 2>&1)" + status=$? + set -e + [[ "$status" == "0" ]] || die "metrics image rpki-client -V failed: image=$METRICS_IMAGE exit=$status output=$output" + grep -q "rpki-client" <<<"$output" || die "metrics image rpki-client -V unexpected output: $output" } verify_image_platform() { diff --git a/docker/ours-rp-metrics.Dockerfile b/docker/ours-rp-metrics.Dockerfile index 97f3fc3..f7abee5 100644 --- a/docker/ours-rp-metrics.Dockerfile +++ b/docker/ours-rp-metrics.Dockerfile @@ -85,27 +85,66 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ mkdir -p /build-out/bin; \ cp "target/$target_triple/release/rpki_artifact_metrics" /build-out/bin/ +FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS rpki-client-builder + +ARG RPKI_CLIENT_VERSION=9.8 +ARG RPKI_CLIENT_SHA256=42920aac5afd0996173fb9f7848691a2c49dd234e4f10478808c1aa475620861 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + libexpat1-dev \ + libssl-dev \ + libtls-dev \ + pkg-config \ + rsync \ + zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --system --no-create-home --shell /usr/sbin/nologin _rpki-client + +RUN set -eux; \ + curl -fsSL "https://ftp.openbsd.org/pub/OpenBSD/rpki-client/rpki-client-${RPKI_CLIENT_VERSION}.tar.gz" -o /tmp/rpki-client.tar.gz; \ + echo "${RPKI_CLIENT_SHA256} /tmp/rpki-client.tar.gz" | sha256sum -c -; \ + tar -xzf /tmp/rpki-client.tar.gz -C /tmp; \ + cd "/tmp/rpki-client-${RPKI_CLIENT_VERSION}"; \ + ./configure --prefix=/opt/rpki-client; \ + make -j"$(nproc)"; \ + make install; \ + /opt/rpki-client/sbin/rpki-client -V + FROM --platform=$TARGETPLATFORM ${RUNTIME_IMAGE} AS runtime RUN apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ + libexpat1 \ + libssl3 \ + libtls26 \ tzdata \ - && rm -rf /var/lib/apt/lists/* + zlib1g \ + && rm -rf /var/lib/apt/lists/* \ + && useradd --system --no-create-home --shell /usr/sbin/nologin _rpki-client \ + && mkdir -p /opt/rpki-client/var/cache/rpki-client WORKDIR /opt/ours-rp COPY --from=builder /build-out/bin/ /opt/ours-rp/bin/ +COPY --from=rpki-client-builder /opt/rpki-client/ /opt/rpki-client/ +COPY --from=rpki-client-builder /opt/rpki-client/sbin/rpki-client /opt/ours-rp/bin/rpki-client ARG SOURCE_COMMIT=unknown ARG SOURCE_DIRTY=unknown ARG BUILD_TIMESTAMP_UTC=unknown +ARG RPKI_CLIENT_VERSION=9.8 LABEL org.opencontainers.image.title="ours-rp-metrics" \ org.opencontainers.image.description="Ours RP artifact metrics image for multi-arch Docker Compose deployment" \ org.opencontainers.image.revision="${SOURCE_COMMIT}" \ org.opencontainers.image.created="${BUILD_TIMESTAMP_UTC}" \ - org.opencontainers.image.source-dirty="${SOURCE_DIRTY}" + org.opencontainers.image.source-dirty="${SOURCE_DIRTY}" \ + org.opencontainers.image.rpki-client.version="${RPKI_CLIENT_VERSION}" RUN chmod +x /opt/ours-rp/bin/* \ && mkdir -p /var/lib/ours-rp/state /var/lib/ours-rp/runs /var/lib/ours-rp/logs diff --git a/src/tools/rpki_artifact_metrics.rs b/src/tools/rpki_artifact_metrics.rs index d2060fa..b435fa9 100644 --- a/src/tools/rpki_artifact_metrics.rs +++ b/src/tools/rpki_artifact_metrics.rs @@ -3,6 +3,7 @@ use std::fs; use std::io::{Read, Write}; use std::net::{TcpListener, TcpStream}; use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; use std::sync::{Arc, RwLock}; use std::thread; use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; @@ -26,10 +27,13 @@ struct Args { once: bool, out_metrics: Option, out_status: Option, + ccr_check_bin: Option, + ccr_check_timeout_secs: u64, + ccr_check_max_per_scan: u64, } fn usage() -> &'static str { - "Usage: rpki_artifact_metrics --run-root [--listen ] [--poll-secs ] [--instance ] [--once] [--out-metrics ] [--out-status ]" + "Usage: rpki_artifact_metrics --run-root [--listen ] [--poll-secs ] [--instance ] [--once] [--out-metrics ] [--out-status ] [--ccr-check-bin ] [--ccr-check-timeout-secs ] [--ccr-check-max-per-scan ]" } pub fn main_entry() -> Result<(), String> { @@ -38,8 +42,16 @@ pub fn main_entry() -> Result<(), String> { fn real_main() -> Result<(), String> { let args = parse_args(&std::env::args().collect::>())?; + let ccr_check_config = CcrCheckConfig::from_args(&args); + let rtr_report_dir = configured_rtr_report_dir(); if args.once { - let snapshot = scan_run_root(&args.run_root, &args.instance)?; + let snapshot = scan_run_root_incremental_with_rtr_dir( + &args.run_root, + &args.instance, + None, + rtr_report_dir.as_deref(), + &ccr_check_config, + )?; let metrics = render_metrics(&snapshot); let status = render_status_json(&snapshot)?; if let Some(path) = args.out_metrics.as_ref() { @@ -53,7 +65,13 @@ fn real_main() -> Result<(), String> { return Ok(()); } - let shared = Arc::new(RwLock::new(scan_run_root(&args.run_root, &args.instance)?)); + let shared = Arc::new(RwLock::new(scan_run_root_incremental_with_rtr_dir( + &args.run_root, + &args.instance, + None, + rtr_report_dir.as_deref(), + &ccr_check_config, + )?)); let scanner = Arc::clone(&shared); let run_root = args.run_root.clone(); let instance = args.instance.clone(); @@ -62,20 +80,25 @@ fn real_main() -> Result<(), String> { loop { thread::sleep(Duration::from_secs(poll_secs)); let previous_snapshot = scanner.read().expect("metrics lock poisoned").clone(); - let next = - match scan_run_root_incremental(&run_root, &instance, Some(&previous_snapshot)) { - Ok(snapshot) => snapshot, - Err(err) => { - let mut previous = scanner.write().expect("metrics lock poisoned"); - previous - .service - .parse_errors - .push(format!("scan failed: {err}")); - previous.service.last_reload_success = false; - previous.service.last_scan_timestamp_seconds = unix_now_seconds(); - continue; - } - }; + let next = match scan_run_root_incremental_with_rtr_dir( + &run_root, + &instance, + Some(&previous_snapshot), + rtr_report_dir.as_deref(), + &ccr_check_config, + ) { + Ok(snapshot) => snapshot, + Err(err) => { + let mut previous = scanner.write().expect("metrics lock poisoned"); + previous + .service + .parse_errors + .push(format!("scan failed: {err}")); + previous.service.last_reload_success = false; + previous.service.last_scan_timestamp_seconds = unix_now_seconds(); + continue; + } + }; *scanner.write().expect("metrics lock poisoned") = next; } }); @@ -91,6 +114,9 @@ fn parse_args(argv: &[String]) -> Result { let mut once = false; let mut out_metrics = None; let mut out_status = None; + let mut ccr_check_bin = None; + let mut ccr_check_timeout_secs = 120u64; + let mut ccr_check_max_per_scan = 2u64; let mut index = 1usize; while index < argv.len() { match argv[index].as_str() { @@ -122,6 +148,24 @@ fn parse_args(argv: &[String]) -> Result { index += 1; out_status = Some(PathBuf::from(value_at(argv, index, "--out-status")?)); } + "--ccr-check-bin" => { + index += 1; + ccr_check_bin = Some(PathBuf::from(value_at(argv, index, "--ccr-check-bin")?)); + } + "--ccr-check-timeout-secs" => { + index += 1; + let value = value_at(argv, index, "--ccr-check-timeout-secs")?; + ccr_check_timeout_secs = value + .parse::() + .map_err(|_| format!("invalid --ccr-check-timeout-secs: {value}"))?; + } + "--ccr-check-max-per-scan" => { + index += 1; + let value = value_at(argv, index, "--ccr-check-max-per-scan")?; + ccr_check_max_per_scan = value + .parse::() + .map_err(|_| format!("invalid --ccr-check-max-per-scan: {value}"))?; + } "-h" | "--help" => return Err(usage().to_string()), other => return Err(format!("unknown argument: {other}\n{}", usage())), } @@ -135,6 +179,9 @@ fn parse_args(argv: &[String]) -> Result { once, out_metrics, out_status, + ccr_check_bin, + ccr_check_timeout_secs, + ccr_check_max_per_scan, }) } @@ -161,9 +208,67 @@ struct MetricsSnapshot { top_pp_by_sync_duration: Vec, cir: Option, ccr: Option, + ccr_check: CcrCheckMetrics, rtr: RtrMetrics, } +#[derive(Clone, Debug, Default, Serialize)] +#[serde(rename_all = "camelCase")] +struct CcrCheckMetrics { + enabled: bool, + up: bool, + bin: Option, + pass_total: u64, + ccr_parse_error_total: u64, + tool_error_total: u64, + consecutive_failures: u64, + last: Option, + checked_run_dirs: Vec, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct CcrCheckLastResult { + run_dir: String, + run_seq: Option, + outcome: String, + duration_seconds: f64, + checked_at_seconds: f64, + detail: Option, +} + +#[derive(Clone, Debug)] +struct CcrCheckConfig { + bin: Option, + timeout_secs: u64, + max_per_scan: u64, +} + +impl CcrCheckConfig { + fn disabled() -> Self { + Self { + bin: None, + timeout_secs: 120, + max_per_scan: 2, + } + } + + fn from_args(args: &Args) -> Self { + let bin = args.ccr_check_bin.clone().or_else(|| { + std::env::var("RPKI_METRICS_CCR_CHECK_BIN") + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + .map(PathBuf::from) + }); + Self { + bin, + timeout_secs: args.ccr_check_timeout_secs, + max_per_scan: args.ccr_check_max_per_scan, + } + } +} + #[derive(Clone, Debug, Default, Serialize)] #[serde(rename_all = "camelCase")] struct ServiceMetrics { @@ -493,21 +598,13 @@ struct RunRecord { } fn scan_run_root(input_root: &Path, instance: &str) -> Result { - let rtr_report_dir = configured_rtr_report_dir(); - scan_run_root_incremental_with_rtr_dir(input_root, instance, None, rtr_report_dir.as_deref()) -} - -fn scan_run_root_incremental( - input_root: &Path, - instance: &str, - previous: Option<&MetricsSnapshot>, -) -> Result { let rtr_report_dir = configured_rtr_report_dir(); scan_run_root_incremental_with_rtr_dir( input_root, instance, - previous, + None, rtr_report_dir.as_deref(), + &CcrCheckConfig::disabled(), ) } @@ -516,6 +613,7 @@ fn scan_run_root_incremental_with_rtr_dir( instance: &str, previous: Option<&MetricsSnapshot>, rtr_report_dir: Option<&Path>, + ccr_check_config: &CcrCheckConfig, ) -> Result { let started = Instant::now(); let runs_root = resolve_runs_root(input_root); @@ -570,6 +668,7 @@ fn scan_run_root_incremental_with_rtr_dir( } snapshot.rtr = scan_rtr_reports(rtr_report_dir, previous.map(|snapshot| &snapshot.rtr)); + run_pending_ccr_checks(&records, previous, ccr_check_config, &mut snapshot); snapshot.service.last_scan_timestamp_seconds = unix_now_seconds(); snapshot.service.last_scan_duration_seconds = started.elapsed().as_secs_f64(); snapshot.service.last_reload_success = @@ -975,6 +1074,218 @@ fn consecutive_failures(records: &[RunRecord]) -> u64 { count } +const MAX_CHECKED_RUN_DIRS: usize = 512; + +fn run_pending_ccr_checks( + records: &[RunRecord], + previous: Option<&MetricsSnapshot>, + config: &CcrCheckConfig, + snapshot: &mut MetricsSnapshot, +) { + let mut check = previous + .map(|prev| prev.ccr_check.clone()) + .unwrap_or_default(); + check.enabled = config.bin.is_some(); + check.bin = config.bin.as_ref().map(|bin| bin.display().to_string()); + check.up = false; + + let Some(bin) = config.bin.as_ref() else { + snapshot.ccr_check = check; + return; + }; + if !bin.is_file() { + snapshot + .service + .parse_errors + .push(format!("ccr check bin is not a file: {}", bin.display())); + snapshot.ccr_check = check; + return; + } + check.up = true; + + let mut pending: Vec<&RunRecord> = records + .iter() + .filter(|record| record.status == "success") + .filter(|record| record.path.join("result.ccr").is_file()) + .filter(|record| { + let dir = record.path.display().to_string(); + !check.checked_run_dirs.iter().any(|seen| seen == &dir) + }) + .collect(); + + if previous.is_none() { + // On service start only the latest existing run is checked; older + // history is marked as seen without executing checks. + let latest_path = pending.last().map(|record| record.path.clone()); + for record in &pending { + if Some(&record.path) != latest_path.as_ref() { + check + .checked_run_dirs + .push(record.path.display().to_string()); + } + } + pending.retain(|record| Some(&record.path) == latest_path.as_ref()); + } + + for record in pending + .into_iter() + .take(config.max_per_scan.max(1) as usize) + { + let ccr_path = record.path.join("result.ccr"); + let outcome = execute_ccr_check(bin, &ccr_path, Duration::from_secs(config.timeout_secs)); + let run_seq = record + .summary + .as_ref() + .and_then(|v| json_u64(v, &["runSeq"])) + .or_else(|| { + record + .meta + .as_ref() + .and_then(|v| json_u64(v, &["run_index"])) + }); + let passed = outcome.outcome == "pass"; + match outcome.outcome.as_str() { + "pass" => check.pass_total += 1, + "ccr_parse_error" => check.ccr_parse_error_total += 1, + _ => check.tool_error_total += 1, + } + check.consecutive_failures = if passed { + 0 + } else { + check.consecutive_failures + 1 + }; + check.last = Some(CcrCheckLastResult { + run_dir: record.path.display().to_string(), + run_seq, + outcome: outcome.outcome, + duration_seconds: outcome.duration_seconds, + checked_at_seconds: unix_now_seconds(), + detail: outcome.detail, + }); + check + .checked_run_dirs + .push(record.path.display().to_string()); + } + if check.checked_run_dirs.len() > MAX_CHECKED_RUN_DIRS { + let drain = check.checked_run_dirs.len() - MAX_CHECKED_RUN_DIRS; + check.checked_run_dirs.drain(..drain); + } + snapshot.ccr_check = check; +} + +struct CcrCheckOutcome { + outcome: String, + duration_seconds: f64, + detail: Option, +} + +fn execute_ccr_check(bin: &Path, ccr_path: &Path, timeout: Duration) -> CcrCheckOutcome { + let started = Instant::now(); + let log_path = std::env::temp_dir().join(format!( + "rpki-artifact-metrics-ccr-check-{}-{}.log", + std::process::id(), + SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_nanos()) + .unwrap_or(0) + )); + let (outcome, detail) = execute_ccr_check_logged(bin, ccr_path, timeout, &log_path); + let _ = fs::remove_file(&log_path); + CcrCheckOutcome { + outcome, + duration_seconds: started.elapsed().as_secs_f64(), + detail, + } +} + +fn execute_ccr_check_logged( + bin: &Path, + ccr_path: &Path, + timeout: Duration, + log_path: &Path, +) -> (String, Option) { + let log_file = match fs::File::create(log_path) { + Ok(file) => file, + Err(err) => { + return ( + "tool_error".to_string(), + Some(format!("create ccr check log failed: {err}")), + ); + } + }; + let log_err = match log_file.try_clone() { + Ok(file) => file, + Err(err) => { + return ( + "tool_error".to_string(), + Some(format!("clone ccr check log handle failed: {err}")), + ); + } + }; + let mut child = match Command::new(bin) + .arg("-f") + .arg(ccr_path) + .stdin(Stdio::null()) + .stdout(Stdio::from(log_file)) + .stderr(Stdio::from(log_err)) + .spawn() + { + Ok(child) => child, + Err(err) => { + return ( + "tool_error".to_string(), + Some(format!("spawn ccr check failed: {err}")), + ); + } + }; + let started = Instant::now(); + let exit_code = loop { + match child.try_wait() { + Ok(Some(status)) => break status.code(), + Ok(None) => { + if started.elapsed() >= timeout { + let _ = child.kill(); + let _ = child.wait(); + return ( + "tool_error".to_string(), + Some(format!("ccr check timed out after {}s", timeout.as_secs())), + ); + } + thread::sleep(Duration::from_millis(100)); + } + Err(err) => { + return ( + "tool_error".to_string(), + Some(format!("wait ccr check failed: {err}")), + ); + } + } + }; + let output = fs::read_to_string(log_path).unwrap_or_default(); + classify_ccr_check_output(exit_code, ccr_path, &output) +} + +fn classify_ccr_check_output( + exit_code: Option, + ccr_path: &Path, + output: &str, +) -> (String, Option) { + if exit_code != Some(0) { + return ( + "tool_error".to_string(), + Some(format!("rpki-client exited with code {exit_code:?}")), + ); + } + // rpki-client exits 0 even when the CCR file itself fails to parse, so + // the file-level error line is the authoritative failure signal. This + // mirrors scripts/ccr/check_rpki_client_ccr.sh. + let marker = format!("rpki-client: {}:", ccr_path.display()); + if let Some(line) = output.lines().find(|line| line.contains(&marker)) { + return ("ccr_parse_error".to_string(), Some(line.trim().to_string())); + } + ("pass".to_string(), None) +} + fn read_json_optional(path: &Path, errors: &mut Vec) -> Option { if !path.exists() { return None; @@ -1634,10 +1945,60 @@ fn render_metrics(snapshot: &MetricsSnapshot) -> String { if let Some(ccr) = snapshot.ccr.as_ref() { render_ccr_metrics(&mut writer, instance, ccr); } + render_ccr_check_metrics(&mut writer, instance, &snapshot.ccr_check); render_rtr_metrics(&mut writer, instance, snapshot); out } +fn render_ccr_check_metrics(writer: &mut PromWriter<'_>, instance: &str, check: &CcrCheckMetrics) { + writer.gauge( + "ours_rp_ccr_format_check_up", + "Whether the CCR rpki-client format checker binary is configured and present", + &[label("instance", instance)], + bool_value(check.up), + ); + for (result, value) in [ + ("pass", check.pass_total), + ("ccr_parse_error", check.ccr_parse_error_total), + ("tool_error", check.tool_error_total), + ] { + writer.counter( + "ours_rp_ccr_format_check_total", + "CCR rpki-client format check results by classification", + &[label("instance", instance), label("result", result)], + value as f64, + ); + } + writer.gauge( + "ours_rp_ccr_format_check_consecutive_failures", + "Consecutive failed CCR rpki-client format checks", + &[label("instance", instance)], + check.consecutive_failures as f64, + ); + if let Some(last) = check.last.as_ref() { + writer.gauge( + "ours_rp_ccr_format_check_last_success", + "Whether the latest CCR rpki-client format check passed", + &[label("instance", instance)], + bool_value(last.outcome == "pass"), + ); + if let Some(run_seq) = last.run_seq { + writer.gauge( + "ours_rp_ccr_format_check_last_run_sequence", + "Run sequence of the latest checked CCR artifact", + &[label("instance", instance)], + run_seq as f64, + ); + } + writer.gauge( + "ours_rp_ccr_format_check_last_duration_seconds", + "Duration of the latest CCR rpki-client format check", + &[label("instance", instance)], + last.duration_seconds, + ); + } +} + fn render_rtr_metrics(writer: &mut PromWriter<'_>, instance: &str, snapshot: &MetricsSnapshot) { let rtr = &snapshot.rtr; let scan_now = snapshot.service.last_scan_timestamp_seconds; @@ -2525,6 +2886,7 @@ fn render_status_json(snapshot: &MetricsSnapshot) -> Result { "latestRun": snapshot.latest_run, "cir": snapshot.cir, "ccr": snapshot.ccr, + "ccrCheck": snapshot.ccr_check, "rtr": snapshot.rtr, "topRepositoriesBySyncDuration": snapshot.top_repos_by_sync_duration, "topPublicationPointsByObjectCount": snapshot.top_pp_by_object_count, @@ -3162,8 +3524,14 @@ mod tests { ) .expect("clients"); - let snapshot = scan_run_root_incremental_with_rtr_dir(td.path(), "test", None, Some(&rtr)) - .expect("scan"); + let snapshot = scan_run_root_incremental_with_rtr_dir( + td.path(), + "test", + None, + Some(&rtr), + &CcrCheckConfig::disabled(), + ) + .expect("scan"); assert!(snapshot.rtr.enabled); assert_eq!(snapshot.rtr.files["source"].seen, 1); assert_eq!( @@ -3233,8 +3601,14 @@ mod tests { ) .expect("clients"); - let snapshot = scan_run_root_incremental_with_rtr_dir(td.path(), "test", None, Some(&rtr)) - .expect("scan"); + let snapshot = scan_run_root_incremental_with_rtr_dir( + td.path(), + "test", + None, + Some(&rtr), + &CcrCheckConfig::disabled(), + ) + .expect("scan"); assert!(snapshot.rtr.enabled); assert_eq!(snapshot.rtr.files["source"].seen, 1); assert!(snapshot.rtr.source.is_none()); @@ -3247,6 +3621,223 @@ mod tests { assert!(metrics.contains(r#"ours_rp_rtr_metrics_enabled{instance="test"} 1"#)); } + #[test] + fn parse_args_accepts_ccr_check_options() { + let args = parse_args(&[ + "rpki_artifact_metrics".to_string(), + "--run-root".to_string(), + "root".to_string(), + "--ccr-check-bin".to_string(), + "/opt/ours-rp/bin/rpki-client".to_string(), + "--ccr-check-timeout-secs".to_string(), + "60".to_string(), + "--ccr-check-max-per-scan".to_string(), + "3".to_string(), + ]) + .expect("parse"); + assert_eq!( + args.ccr_check_bin.as_deref(), + Some(Path::new("/opt/ours-rp/bin/rpki-client")) + ); + assert_eq!(args.ccr_check_timeout_secs, 60); + assert_eq!(args.ccr_check_max_per_scan, 3); + } + + #[test] + fn ccr_check_disabled_without_bin() { + let td = TempDir::new().expect("tempdir"); + write_success_run(td.path(), "run_0001", 1); + let snapshot = scan_run_root(td.path(), "test").expect("scan"); + assert!(!snapshot.ccr_check.enabled); + assert!(!snapshot.ccr_check.up); + assert!(snapshot.ccr_check.last.is_none()); + let metrics = render_metrics(&snapshot); + assert!(metrics.contains(r#"ours_rp_ccr_format_check_up{instance="test"} 0"#)); + assert!(!metrics.contains("ours_rp_ccr_format_check_last_success")); + } + + #[test] + fn ccr_check_pass_records_metrics() { + let td = TempDir::new().expect("tempdir"); + write_success_run(td.path(), "run_0001", 1); + let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 0\n"); + let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30); + assert!(snapshot.ccr_check.up); + assert_eq!(snapshot.ccr_check.pass_total, 1); + assert_eq!(snapshot.ccr_check.consecutive_failures, 0); + let last = snapshot.ccr_check.last.as_ref().expect("last result"); + assert_eq!(last.outcome, "pass"); + assert_eq!(last.run_seq, Some(1)); + let metrics = render_metrics(&snapshot); + assert!( + metrics.contains(r#"ours_rp_ccr_format_check_total{instance="test",result="pass"} 1"#) + ); + assert!(metrics.contains(r#"ours_rp_ccr_format_check_last_success{instance="test"} 1"#)); + assert!( + metrics.contains(r#"ours_rp_ccr_format_check_last_run_sequence{instance="test"} 1"#) + ); + let status = render_status_json(&snapshot).expect("status"); + assert!(status.contains(r#""ccrCheck""#)); + assert!(status.contains(r#""passTotal": 1"#)); + } + + #[test] + fn ccr_check_parse_error_detected_from_output() { + let td = TempDir::new().expect("tempdir"); + write_success_run(td.path(), "run_0001", 1); + let bin = write_fake_rpki_client( + td.path(), + "#!/bin/sh\necho \"rpki-client: $2: unexpected number of locations\" >&2\nexit 0\n", + ); + let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30); + assert_eq!(snapshot.ccr_check.ccr_parse_error_total, 1); + assert_eq!(snapshot.ccr_check.consecutive_failures, 1); + let last = snapshot.ccr_check.last.as_ref().expect("last result"); + assert_eq!(last.outcome, "ccr_parse_error"); + assert!( + last.detail + .as_deref() + .unwrap_or_default() + .contains("unexpected number of locations") + ); + let metrics = render_metrics(&snapshot); + assert!(metrics.contains( + r#"ours_rp_ccr_format_check_total{instance="test",result="ccr_parse_error"} 1"# + )); + assert!(metrics.contains(r#"ours_rp_ccr_format_check_last_success{instance="test"} 0"#)); + } + + #[test] + fn ccr_check_nonzero_exit_is_tool_error() { + let td = TempDir::new().expect("tempdir"); + write_success_run(td.path(), "run_0001", 1); + let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 3\n"); + let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30); + assert_eq!(snapshot.ccr_check.tool_error_total, 1); + assert_eq!( + snapshot.ccr_check.last.as_ref().expect("last").outcome, + "tool_error" + ); + } + + #[test] + fn ccr_check_timeout_is_tool_error() { + let td = TempDir::new().expect("tempdir"); + write_success_run(td.path(), "run_0001", 1); + let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nsleep 30\n"); + let snapshot = scan_with_ccr_check(td.path(), None, &bin, 1); + let last = snapshot.ccr_check.last.as_ref().expect("last result"); + assert_eq!(last.outcome, "tool_error"); + assert!( + last.detail + .as_deref() + .unwrap_or_default() + .contains("timed out") + ); + } + + #[test] + fn ccr_check_startup_only_checks_latest_and_new_runs_once() { + let td = TempDir::new().expect("tempdir"); + write_success_run(td.path(), "run_0001", 1); + write_success_run(td.path(), "run_0002", 2); + let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 0\n"); + + // First scan at service start: only the latest existing run is + // checked, older history is marked as seen without executing checks. + let first = scan_with_ccr_check(td.path(), None, &bin, 30); + assert_eq!(first.ccr_check.pass_total, 1); + assert!( + first + .ccr_check + .last + .as_ref() + .expect("last") + .run_dir + .contains("run_0002") + ); + + // Second scan: pre-existing history must not be backfilled. + let second = scan_with_ccr_check(td.path(), Some(&first), &bin, 30); + assert_eq!(second.ccr_check.pass_total, 1); + + // A newly completed run is checked exactly once. + write_success_run(td.path(), "run_0003", 3); + let third = scan_with_ccr_check(td.path(), Some(&second), &bin, 30); + assert_eq!(third.ccr_check.pass_total, 2); + assert_eq!( + third.ccr_check.last.as_ref().expect("last").run_seq, + Some(3) + ); + let fourth = scan_with_ccr_check(td.path(), Some(&third), &bin, 30); + assert_eq!(fourth.ccr_check.pass_total, 2); + } + + #[test] + fn ccr_check_missing_bin_marks_not_up() { + let td = TempDir::new().expect("tempdir"); + write_success_run(td.path(), "run_0001", 1); + let missing = td.path().join("no-such-rpki-client"); + let snapshot = scan_with_ccr_check(td.path(), None, &missing, 30); + assert!(snapshot.ccr_check.enabled); + assert!(!snapshot.ccr_check.up); + assert!(snapshot.ccr_check.last.is_none()); + assert!( + snapshot + .service + .parse_errors + .iter() + .any(|error| error.contains("ccr check bin is not a file")) + ); + let metrics = render_metrics(&snapshot); + assert!(metrics.contains(r#"ours_rp_ccr_format_check_up{instance="test"} 0"#)); + } + + fn write_success_run(root: &Path, name: &str, seq: u64) { + let run = root.join("runs").join(name); + fs::create_dir_all(&run).expect("create run"); + fs::write( + run.join("run-meta.json"), + format!( + r#"{{"status":"success","run_index":{seq},"run_id":"{name}","sync_mode":"snapshot","snapshot_reason":"first_run","started_at_rfc3339_utc":"2026-07-21T00:00:00Z","completed_at_rfc3339_utc":"2026-07-21T00:00:10Z"}}"# + ), + ) + .expect("meta"); + fs::write( + run.join("run-summary.json"), + format!( + r#"{{"runSeq":{seq},"runId":"{name}","startedAtRfc3339Utc":"2026-07-21T00:00:00Z","finishedAtRfc3339Utc":"2026-07-21T00:00:10Z","wallMs":10000,"status":"success","exitCode":0}}"# + ), + ) + .expect("summary"); + fs::write(run.join("result.ccr"), sample_ccr()).expect("ccr"); + } + + fn write_fake_rpki_client(root: &Path, body: &str) -> PathBuf { + let path = root.join("fake-rpki-client.sh"); + fs::write(&path, body).expect("write fake rpki-client"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod"); + } + path + } + + fn scan_with_ccr_check( + root: &Path, + previous: Option<&MetricsSnapshot>, + bin: &Path, + timeout_secs: u64, + ) -> MetricsSnapshot { + let config = CcrCheckConfig { + bin: Some(bin.to_path_buf()), + timeout_secs, + max_per_scan: 2, + }; + scan_run_root_incremental_with_rtr_dir(root, "test", previous, None, &config).expect("scan") + } + fn sample_cir() -> Vec { let rejected = vec![CirRejectedObject { object_uri: "rsync://repo.example/a/bad.roa".to_string(),