20260721 metrics 服务新增 CCR rpki-client 格式检查指标与 Grafana 面板（#125）

src/tools/rpki_artifact_metrics.rs

@@ -3,6 +3,7 @@ use std::fs;
 use std::io::{Read, Write};
 use std::net::{TcpListener, TcpStream};
 use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
 use std::sync::{Arc, RwLock};
 use std::thread;
 use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
@@ -26,10 +27,13 @@ struct Args {
     once: bool,
     out_metrics: Option<PathBuf>,
     out_status: Option<PathBuf>,
+    ccr_check_bin: Option<PathBuf>,
+    ccr_check_timeout_secs: u64,
+    ccr_check_max_per_scan: u64,
 }
 
 fn usage() -> &'static str {
-    "Usage: rpki_artifact_metrics --run-root <path> [--listen <addr:port>] [--poll-secs <n>] [--instance <name>] [--once] [--out-metrics <path>] [--out-status <path>]"
+    "Usage: rpki_artifact_metrics --run-root <path> [--listen <addr:port>] [--poll-secs <n>] [--instance <name>] [--once] [--out-metrics <path>] [--out-status <path>] [--ccr-check-bin <path>] [--ccr-check-timeout-secs <n>] [--ccr-check-max-per-scan <n>]"
 }
 
 pub fn main_entry() -> Result<(), String> {
@@ -38,8 +42,16 @@ pub fn main_entry() -> Result<(), String> {
 
 fn real_main() -> Result<(), String> {
     let args = parse_args(&std::env::args().collect::<Vec<_>>())?;
+    let ccr_check_config = CcrCheckConfig::from_args(&args);
+    let rtr_report_dir = configured_rtr_report_dir();
     if args.once {
-        let snapshot = scan_run_root(&args.run_root, &args.instance)?;
+        let snapshot = scan_run_root_incremental_with_rtr_dir(
+            &args.run_root,
+            &args.instance,
+            None,
+            rtr_report_dir.as_deref(),
+            &ccr_check_config,
+        )?;
         let metrics = render_metrics(&snapshot);
         let status = render_status_json(&snapshot)?;
         if let Some(path) = args.out_metrics.as_ref() {
@@ -53,7 +65,13 @@ fn real_main() -> Result<(), String> {
         return Ok(());
     }
 
-    let shared = Arc::new(RwLock::new(scan_run_root(&args.run_root, &args.instance)?));
+    let shared = Arc::new(RwLock::new(scan_run_root_incremental_with_rtr_dir(
+        &args.run_root,
+        &args.instance,
+        None,
+        rtr_report_dir.as_deref(),
+        &ccr_check_config,
+    )?));
     let scanner = Arc::clone(&shared);
     let run_root = args.run_root.clone();
     let instance = args.instance.clone();
@@ -62,20 +80,25 @@ fn real_main() -> Result<(), String> {
         loop {
             thread::sleep(Duration::from_secs(poll_secs));
             let previous_snapshot = scanner.read().expect("metrics lock poisoned").clone();
-            let next =
-                match scan_run_root_incremental(&run_root, &instance, Some(&previous_snapshot)) {
-                    Ok(snapshot) => snapshot,
-                    Err(err) => {
-                        let mut previous = scanner.write().expect("metrics lock poisoned");
-                        previous
-                            .service
-                            .parse_errors
-                            .push(format!("scan failed: {err}"));
-                        previous.service.last_reload_success = false;
-                        previous.service.last_scan_timestamp_seconds = unix_now_seconds();
-                        continue;
-                    }
-                };
+            let next = match scan_run_root_incremental_with_rtr_dir(
+                &run_root,
+                &instance,
+                Some(&previous_snapshot),
+                rtr_report_dir.as_deref(),
+                &ccr_check_config,
+            ) {
+                Ok(snapshot) => snapshot,
+                Err(err) => {
+                    let mut previous = scanner.write().expect("metrics lock poisoned");
+                    previous
+                        .service
+                        .parse_errors
+                        .push(format!("scan failed: {err}"));
+                    previous.service.last_reload_success = false;
+                    previous.service.last_scan_timestamp_seconds = unix_now_seconds();
+                    continue;
+                }
+            };
             *scanner.write().expect("metrics lock poisoned") = next;
         }
     });
@@ -91,6 +114,9 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
     let mut once = false;
     let mut out_metrics = None;
     let mut out_status = None;
+    let mut ccr_check_bin = None;
+    let mut ccr_check_timeout_secs = 120u64;
+    let mut ccr_check_max_per_scan = 2u64;
     let mut index = 1usize;
     while index < argv.len() {
         match argv[index].as_str() {
@@ -122,6 +148,24 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
                 index += 1;
                 out_status = Some(PathBuf::from(value_at(argv, index, "--out-status")?));
             }
+            "--ccr-check-bin" => {
+                index += 1;
+                ccr_check_bin = Some(PathBuf::from(value_at(argv, index, "--ccr-check-bin")?));
+            }
+            "--ccr-check-timeout-secs" => {
+                index += 1;
+                let value = value_at(argv, index, "--ccr-check-timeout-secs")?;
+                ccr_check_timeout_secs = value
+                    .parse::<u64>()
+                    .map_err(|_| format!("invalid --ccr-check-timeout-secs: {value}"))?;
+            }
+            "--ccr-check-max-per-scan" => {
+                index += 1;
+                let value = value_at(argv, index, "--ccr-check-max-per-scan")?;
+                ccr_check_max_per_scan = value
+                    .parse::<u64>()
+                    .map_err(|_| format!("invalid --ccr-check-max-per-scan: {value}"))?;
+            }
             "-h" | "--help" => return Err(usage().to_string()),
             other => return Err(format!("unknown argument: {other}\n{}", usage())),
         }
@@ -135,6 +179,9 @@ fn parse_args(argv: &[String]) -> Result<Args, String> {
         once,
         out_metrics,
         out_status,
+        ccr_check_bin,
+        ccr_check_timeout_secs,
+        ccr_check_max_per_scan,
     })
 }
 
@@ -161,9 +208,67 @@ struct MetricsSnapshot {
     top_pp_by_sync_duration: Vec<TopPublicationPoint>,
     cir: Option<CirMetrics>,
     ccr: Option<CcrMetrics>,
+    ccr_check: CcrCheckMetrics,
     rtr: RtrMetrics,
 }
 
+#[derive(Clone, Debug, Default, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct CcrCheckMetrics {
+    enabled: bool,
+    up: bool,
+    bin: Option<String>,
+    pass_total: u64,
+    ccr_parse_error_total: u64,
+    tool_error_total: u64,
+    consecutive_failures: u64,
+    last: Option<CcrCheckLastResult>,
+    checked_run_dirs: Vec<String>,
+}
+
+#[derive(Clone, Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct CcrCheckLastResult {
+    run_dir: String,
+    run_seq: Option<u64>,
+    outcome: String,
+    duration_seconds: f64,
+    checked_at_seconds: f64,
+    detail: Option<String>,
+}
+
+#[derive(Clone, Debug)]
+struct CcrCheckConfig {
+    bin: Option<PathBuf>,
+    timeout_secs: u64,
+    max_per_scan: u64,
+}
+
+impl CcrCheckConfig {
+    fn disabled() -> Self {
+        Self {
+            bin: None,
+            timeout_secs: 120,
+            max_per_scan: 2,
+        }
+    }
+
+    fn from_args(args: &Args) -> Self {
+        let bin = args.ccr_check_bin.clone().or_else(|| {
+            std::env::var("RPKI_METRICS_CCR_CHECK_BIN")
+                .ok()
+                .map(|value| value.trim().to_string())
+                .filter(|value| !value.is_empty())
+                .map(PathBuf::from)
+        });
+        Self {
+            bin,
+            timeout_secs: args.ccr_check_timeout_secs,
+            max_per_scan: args.ccr_check_max_per_scan,
+        }
+    }
+}
+
 #[derive(Clone, Debug, Default, Serialize)]
 #[serde(rename_all = "camelCase")]
 struct ServiceMetrics {
@@ -493,21 +598,13 @@ struct RunRecord {
 }
 
 fn scan_run_root(input_root: &Path, instance: &str) -> Result<MetricsSnapshot, String> {
-    let rtr_report_dir = configured_rtr_report_dir();
-    scan_run_root_incremental_with_rtr_dir(input_root, instance, None, rtr_report_dir.as_deref())
-}
-
-fn scan_run_root_incremental(
-    input_root: &Path,
-    instance: &str,
-    previous: Option<&MetricsSnapshot>,
-) -> Result<MetricsSnapshot, String> {
     let rtr_report_dir = configured_rtr_report_dir();
     scan_run_root_incremental_with_rtr_dir(
         input_root,
         instance,
-        previous,
+        None,
         rtr_report_dir.as_deref(),
+        &CcrCheckConfig::disabled(),
     )
 }
 
@@ -516,6 +613,7 @@ fn scan_run_root_incremental_with_rtr_dir(
     instance: &str,
     previous: Option<&MetricsSnapshot>,
     rtr_report_dir: Option<&Path>,
+    ccr_check_config: &CcrCheckConfig,
 ) -> Result<MetricsSnapshot, String> {
     let started = Instant::now();
     let runs_root = resolve_runs_root(input_root);
@@ -570,6 +668,7 @@ fn scan_run_root_incremental_with_rtr_dir(
     }
 
     snapshot.rtr = scan_rtr_reports(rtr_report_dir, previous.map(|snapshot| &snapshot.rtr));
+    run_pending_ccr_checks(&records, previous, ccr_check_config, &mut snapshot);
     snapshot.service.last_scan_timestamp_seconds = unix_now_seconds();
     snapshot.service.last_scan_duration_seconds = started.elapsed().as_secs_f64();
     snapshot.service.last_reload_success =
@@ -975,6 +1074,218 @@ fn consecutive_failures(records: &[RunRecord]) -> u64 {
     count
 }
 
+const MAX_CHECKED_RUN_DIRS: usize = 512;
+
+fn run_pending_ccr_checks(
+    records: &[RunRecord],
+    previous: Option<&MetricsSnapshot>,
+    config: &CcrCheckConfig,
+    snapshot: &mut MetricsSnapshot,
+) {
+    let mut check = previous
+        .map(|prev| prev.ccr_check.clone())
+        .unwrap_or_default();
+    check.enabled = config.bin.is_some();
+    check.bin = config.bin.as_ref().map(|bin| bin.display().to_string());
+    check.up = false;
+
+    let Some(bin) = config.bin.as_ref() else {
+        snapshot.ccr_check = check;
+        return;
+    };
+    if !bin.is_file() {
+        snapshot
+            .service
+            .parse_errors
+            .push(format!("ccr check bin is not a file: {}", bin.display()));
+        snapshot.ccr_check = check;
+        return;
+    }
+    check.up = true;
+
+    let mut pending: Vec<&RunRecord> = records
+        .iter()
+        .filter(|record| record.status == "success")
+        .filter(|record| record.path.join("result.ccr").is_file())
+        .filter(|record| {
+            let dir = record.path.display().to_string();
+            !check.checked_run_dirs.iter().any(|seen| seen == &dir)
+        })
+        .collect();
+
+    if previous.is_none() {
+        // On service start only the latest existing run is checked; older
+        // history is marked as seen without executing checks.
+        let latest_path = pending.last().map(|record| record.path.clone());
+        for record in &pending {
+            if Some(&record.path) != latest_path.as_ref() {
+                check
+                    .checked_run_dirs
+                    .push(record.path.display().to_string());
+            }
+        }
+        pending.retain(|record| Some(&record.path) == latest_path.as_ref());
+    }
+
+    for record in pending
+        .into_iter()
+        .take(config.max_per_scan.max(1) as usize)
+    {
+        let ccr_path = record.path.join("result.ccr");
+        let outcome = execute_ccr_check(bin, &ccr_path, Duration::from_secs(config.timeout_secs));
+        let run_seq = record
+            .summary
+            .as_ref()
+            .and_then(|v| json_u64(v, &["runSeq"]))
+            .or_else(|| {
+                record
+                    .meta
+                    .as_ref()
+                    .and_then(|v| json_u64(v, &["run_index"]))
+            });
+        let passed = outcome.outcome == "pass";
+        match outcome.outcome.as_str() {
+            "pass" => check.pass_total += 1,
+            "ccr_parse_error" => check.ccr_parse_error_total += 1,
+            _ => check.tool_error_total += 1,
+        }
+        check.consecutive_failures = if passed {
+            0
+        } else {
+            check.consecutive_failures + 1
+        };
+        check.last = Some(CcrCheckLastResult {
+            run_dir: record.path.display().to_string(),
+            run_seq,
+            outcome: outcome.outcome,
+            duration_seconds: outcome.duration_seconds,
+            checked_at_seconds: unix_now_seconds(),
+            detail: outcome.detail,
+        });
+        check
+            .checked_run_dirs
+            .push(record.path.display().to_string());
+    }
+    if check.checked_run_dirs.len() > MAX_CHECKED_RUN_DIRS {
+        let drain = check.checked_run_dirs.len() - MAX_CHECKED_RUN_DIRS;
+        check.checked_run_dirs.drain(..drain);
+    }
+    snapshot.ccr_check = check;
+}
+
+struct CcrCheckOutcome {
+    outcome: String,
+    duration_seconds: f64,
+    detail: Option<String>,
+}
+
+fn execute_ccr_check(bin: &Path, ccr_path: &Path, timeout: Duration) -> CcrCheckOutcome {
+    let started = Instant::now();
+    let log_path = std::env::temp_dir().join(format!(
+        "rpki-artifact-metrics-ccr-check-{}-{}.log",
+        std::process::id(),
+        SystemTime::now()
+            .duration_since(UNIX_EPOCH)
+            .map(|d| d.as_nanos())
+            .unwrap_or(0)
+    ));
+    let (outcome, detail) = execute_ccr_check_logged(bin, ccr_path, timeout, &log_path);
+    let _ = fs::remove_file(&log_path);
+    CcrCheckOutcome {
+        outcome,
+        duration_seconds: started.elapsed().as_secs_f64(),
+        detail,
+    }
+}
+
+fn execute_ccr_check_logged(
+    bin: &Path,
+    ccr_path: &Path,
+    timeout: Duration,
+    log_path: &Path,
+) -> (String, Option<String>) {
+    let log_file = match fs::File::create(log_path) {
+        Ok(file) => file,
+        Err(err) => {
+            return (
+                "tool_error".to_string(),
+                Some(format!("create ccr check log failed: {err}")),
+            );
+        }
+    };
+    let log_err = match log_file.try_clone() {
+        Ok(file) => file,
+        Err(err) => {
+            return (
+                "tool_error".to_string(),
+                Some(format!("clone ccr check log handle failed: {err}")),
+            );
+        }
+    };
+    let mut child = match Command::new(bin)
+        .arg("-f")
+        .arg(ccr_path)
+        .stdin(Stdio::null())
+        .stdout(Stdio::from(log_file))
+        .stderr(Stdio::from(log_err))
+        .spawn()
+    {
+        Ok(child) => child,
+        Err(err) => {
+            return (
+                "tool_error".to_string(),
+                Some(format!("spawn ccr check failed: {err}")),
+            );
+        }
+    };
+    let started = Instant::now();
+    let exit_code = loop {
+        match child.try_wait() {
+            Ok(Some(status)) => break status.code(),
+            Ok(None) => {
+                if started.elapsed() >= timeout {
+                    let _ = child.kill();
+                    let _ = child.wait();
+                    return (
+                        "tool_error".to_string(),
+                        Some(format!("ccr check timed out after {}s", timeout.as_secs())),
+                    );
+                }
+                thread::sleep(Duration::from_millis(100));
+            }
+            Err(err) => {
+                return (
+                    "tool_error".to_string(),
+                    Some(format!("wait ccr check failed: {err}")),
+                );
+            }
+        }
+    };
+    let output = fs::read_to_string(log_path).unwrap_or_default();
+    classify_ccr_check_output(exit_code, ccr_path, &output)
+}
+
+fn classify_ccr_check_output(
+    exit_code: Option<i32>,
+    ccr_path: &Path,
+    output: &str,
+) -> (String, Option<String>) {
+    if exit_code != Some(0) {
+        return (
+            "tool_error".to_string(),
+            Some(format!("rpki-client exited with code {exit_code:?}")),
+        );
+    }
+    // rpki-client exits 0 even when the CCR file itself fails to parse, so
+    // the file-level error line is the authoritative failure signal. This
+    // mirrors scripts/ccr/check_rpki_client_ccr.sh.
+    let marker = format!("rpki-client: {}:", ccr_path.display());
+    if let Some(line) = output.lines().find(|line| line.contains(&marker)) {
+        return ("ccr_parse_error".to_string(), Some(line.trim().to_string()));
+    }
+    ("pass".to_string(), None)
+}
+
 fn read_json_optional(path: &Path, errors: &mut Vec<String>) -> Option<Value> {
     if !path.exists() {
         return None;
@@ -1634,10 +1945,60 @@ fn render_metrics(snapshot: &MetricsSnapshot) -> String {
     if let Some(ccr) = snapshot.ccr.as_ref() {
         render_ccr_metrics(&mut writer, instance, ccr);
     }
+    render_ccr_check_metrics(&mut writer, instance, &snapshot.ccr_check);
     render_rtr_metrics(&mut writer, instance, snapshot);
     out
 }
 
+fn render_ccr_check_metrics(writer: &mut PromWriter<'_>, instance: &str, check: &CcrCheckMetrics) {
+    writer.gauge(
+        "ours_rp_ccr_format_check_up",
+        "Whether the CCR rpki-client format checker binary is configured and present",
+        &[label("instance", instance)],
+        bool_value(check.up),
+    );
+    for (result, value) in [
+        ("pass", check.pass_total),
+        ("ccr_parse_error", check.ccr_parse_error_total),
+        ("tool_error", check.tool_error_total),
+    ] {
+        writer.counter(
+            "ours_rp_ccr_format_check_total",
+            "CCR rpki-client format check results by classification",
+            &[label("instance", instance), label("result", result)],
+            value as f64,
+        );
+    }
+    writer.gauge(
+        "ours_rp_ccr_format_check_consecutive_failures",
+        "Consecutive failed CCR rpki-client format checks",
+        &[label("instance", instance)],
+        check.consecutive_failures as f64,
+    );
+    if let Some(last) = check.last.as_ref() {
+        writer.gauge(
+            "ours_rp_ccr_format_check_last_success",
+            "Whether the latest CCR rpki-client format check passed",
+            &[label("instance", instance)],
+            bool_value(last.outcome == "pass"),
+        );
+        if let Some(run_seq) = last.run_seq {
+            writer.gauge(
+                "ours_rp_ccr_format_check_last_run_sequence",
+                "Run sequence of the latest checked CCR artifact",
+                &[label("instance", instance)],
+                run_seq as f64,
+            );
+        }
+        writer.gauge(
+            "ours_rp_ccr_format_check_last_duration_seconds",
+            "Duration of the latest CCR rpki-client format check",
+            &[label("instance", instance)],
+            last.duration_seconds,
+        );
+    }
+}
+
 fn render_rtr_metrics(writer: &mut PromWriter<'_>, instance: &str, snapshot: &MetricsSnapshot) {
     let rtr = &snapshot.rtr;
     let scan_now = snapshot.service.last_scan_timestamp_seconds;
@@ -2525,6 +2886,7 @@ fn render_status_json(snapshot: &MetricsSnapshot) -> Result<String, String> {
         "latestRun": snapshot.latest_run,
         "cir": snapshot.cir,
         "ccr": snapshot.ccr,
+        "ccrCheck": snapshot.ccr_check,
         "rtr": snapshot.rtr,
         "topRepositoriesBySyncDuration": snapshot.top_repos_by_sync_duration,
         "topPublicationPointsByObjectCount": snapshot.top_pp_by_object_count,
@@ -3162,8 +3524,14 @@ mod tests {
         )
         .expect("clients");
 
-        let snapshot = scan_run_root_incremental_with_rtr_dir(td.path(), "test", None, Some(&rtr))
-            .expect("scan");
+        let snapshot = scan_run_root_incremental_with_rtr_dir(
+            td.path(),
+            "test",
+            None,
+            Some(&rtr),
+            &CcrCheckConfig::disabled(),
+        )
+        .expect("scan");
         assert!(snapshot.rtr.enabled);
         assert_eq!(snapshot.rtr.files["source"].seen, 1);
         assert_eq!(
@@ -3233,8 +3601,14 @@ mod tests {
         )
         .expect("clients");
 
-        let snapshot = scan_run_root_incremental_with_rtr_dir(td.path(), "test", None, Some(&rtr))
-            .expect("scan");
+        let snapshot = scan_run_root_incremental_with_rtr_dir(
+            td.path(),
+            "test",
+            None,
+            Some(&rtr),
+            &CcrCheckConfig::disabled(),
+        )
+        .expect("scan");
         assert!(snapshot.rtr.enabled);
         assert_eq!(snapshot.rtr.files["source"].seen, 1);
         assert!(snapshot.rtr.source.is_none());
@@ -3247,6 +3621,223 @@ mod tests {
         assert!(metrics.contains(r#"ours_rp_rtr_metrics_enabled{instance="test"} 1"#));
     }
 
+    #[test]
+    fn parse_args_accepts_ccr_check_options() {
+        let args = parse_args(&[
+            "rpki_artifact_metrics".to_string(),
+            "--run-root".to_string(),
+            "root".to_string(),
+            "--ccr-check-bin".to_string(),
+            "/opt/ours-rp/bin/rpki-client".to_string(),
+            "--ccr-check-timeout-secs".to_string(),
+            "60".to_string(),
+            "--ccr-check-max-per-scan".to_string(),
+            "3".to_string(),
+        ])
+        .expect("parse");
+        assert_eq!(
+            args.ccr_check_bin.as_deref(),
+            Some(Path::new("/opt/ours-rp/bin/rpki-client"))
+        );
+        assert_eq!(args.ccr_check_timeout_secs, 60);
+        assert_eq!(args.ccr_check_max_per_scan, 3);
+    }
+
+    #[test]
+    fn ccr_check_disabled_without_bin() {
+        let td = TempDir::new().expect("tempdir");
+        write_success_run(td.path(), "run_0001", 1);
+        let snapshot = scan_run_root(td.path(), "test").expect("scan");
+        assert!(!snapshot.ccr_check.enabled);
+        assert!(!snapshot.ccr_check.up);
+        assert!(snapshot.ccr_check.last.is_none());
+        let metrics = render_metrics(&snapshot);
+        assert!(metrics.contains(r#"ours_rp_ccr_format_check_up{instance="test"} 0"#));
+        assert!(!metrics.contains("ours_rp_ccr_format_check_last_success"));
+    }
+
+    #[test]
+    fn ccr_check_pass_records_metrics() {
+        let td = TempDir::new().expect("tempdir");
+        write_success_run(td.path(), "run_0001", 1);
+        let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 0\n");
+        let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30);
+        assert!(snapshot.ccr_check.up);
+        assert_eq!(snapshot.ccr_check.pass_total, 1);
+        assert_eq!(snapshot.ccr_check.consecutive_failures, 0);
+        let last = snapshot.ccr_check.last.as_ref().expect("last result");
+        assert_eq!(last.outcome, "pass");
+        assert_eq!(last.run_seq, Some(1));
+        let metrics = render_metrics(&snapshot);
+        assert!(
+            metrics.contains(r#"ours_rp_ccr_format_check_total{instance="test",result="pass"} 1"#)
+        );
+        assert!(metrics.contains(r#"ours_rp_ccr_format_check_last_success{instance="test"} 1"#));
+        assert!(
+            metrics.contains(r#"ours_rp_ccr_format_check_last_run_sequence{instance="test"} 1"#)
+        );
+        let status = render_status_json(&snapshot).expect("status");
+        assert!(status.contains(r#""ccrCheck""#));
+        assert!(status.contains(r#""passTotal": 1"#));
+    }
+
+    #[test]
+    fn ccr_check_parse_error_detected_from_output() {
+        let td = TempDir::new().expect("tempdir");
+        write_success_run(td.path(), "run_0001", 1);
+        let bin = write_fake_rpki_client(
+            td.path(),
+            "#!/bin/sh\necho \"rpki-client: $2: unexpected number of locations\" >&2\nexit 0\n",
+        );
+        let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30);
+        assert_eq!(snapshot.ccr_check.ccr_parse_error_total, 1);
+        assert_eq!(snapshot.ccr_check.consecutive_failures, 1);
+        let last = snapshot.ccr_check.last.as_ref().expect("last result");
+        assert_eq!(last.outcome, "ccr_parse_error");
+        assert!(
+            last.detail
+                .as_deref()
+                .unwrap_or_default()
+                .contains("unexpected number of locations")
+        );
+        let metrics = render_metrics(&snapshot);
+        assert!(metrics.contains(
+            r#"ours_rp_ccr_format_check_total{instance="test",result="ccr_parse_error"} 1"#
+        ));
+        assert!(metrics.contains(r#"ours_rp_ccr_format_check_last_success{instance="test"} 0"#));
+    }
+
+    #[test]
+    fn ccr_check_nonzero_exit_is_tool_error() {
+        let td = TempDir::new().expect("tempdir");
+        write_success_run(td.path(), "run_0001", 1);
+        let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 3\n");
+        let snapshot = scan_with_ccr_check(td.path(), None, &bin, 30);
+        assert_eq!(snapshot.ccr_check.tool_error_total, 1);
+        assert_eq!(
+            snapshot.ccr_check.last.as_ref().expect("last").outcome,
+            "tool_error"
+        );
+    }
+
+    #[test]
+    fn ccr_check_timeout_is_tool_error() {
+        let td = TempDir::new().expect("tempdir");
+        write_success_run(td.path(), "run_0001", 1);
+        let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nsleep 30\n");
+        let snapshot = scan_with_ccr_check(td.path(), None, &bin, 1);
+        let last = snapshot.ccr_check.last.as_ref().expect("last result");
+        assert_eq!(last.outcome, "tool_error");
+        assert!(
+            last.detail
+                .as_deref()
+                .unwrap_or_default()
+                .contains("timed out")
+        );
+    }
+
+    #[test]
+    fn ccr_check_startup_only_checks_latest_and_new_runs_once() {
+        let td = TempDir::new().expect("tempdir");
+        write_success_run(td.path(), "run_0001", 1);
+        write_success_run(td.path(), "run_0002", 2);
+        let bin = write_fake_rpki_client(td.path(), "#!/bin/sh\nexit 0\n");
+
+        // First scan at service start: only the latest existing run is
+        // checked, older history is marked as seen without executing checks.
+        let first = scan_with_ccr_check(td.path(), None, &bin, 30);
+        assert_eq!(first.ccr_check.pass_total, 1);
+        assert!(
+            first
+                .ccr_check
+                .last
+                .as_ref()
+                .expect("last")
+                .run_dir
+                .contains("run_0002")
+        );
+
+        // Second scan: pre-existing history must not be backfilled.
+        let second = scan_with_ccr_check(td.path(), Some(&first), &bin, 30);
+        assert_eq!(second.ccr_check.pass_total, 1);
+
+        // A newly completed run is checked exactly once.
+        write_success_run(td.path(), "run_0003", 3);
+        let third = scan_with_ccr_check(td.path(), Some(&second), &bin, 30);
+        assert_eq!(third.ccr_check.pass_total, 2);
+        assert_eq!(
+            third.ccr_check.last.as_ref().expect("last").run_seq,
+            Some(3)
+        );
+        let fourth = scan_with_ccr_check(td.path(), Some(&third), &bin, 30);
+        assert_eq!(fourth.ccr_check.pass_total, 2);
+    }
+
+    #[test]
+    fn ccr_check_missing_bin_marks_not_up() {
+        let td = TempDir::new().expect("tempdir");
+        write_success_run(td.path(), "run_0001", 1);
+        let missing = td.path().join("no-such-rpki-client");
+        let snapshot = scan_with_ccr_check(td.path(), None, &missing, 30);
+        assert!(snapshot.ccr_check.enabled);
+        assert!(!snapshot.ccr_check.up);
+        assert!(snapshot.ccr_check.last.is_none());
+        assert!(
+            snapshot
+                .service
+                .parse_errors
+                .iter()
+                .any(|error| error.contains("ccr check bin is not a file"))
+        );
+        let metrics = render_metrics(&snapshot);
+        assert!(metrics.contains(r#"ours_rp_ccr_format_check_up{instance="test"} 0"#));
+    }
+
+    fn write_success_run(root: &Path, name: &str, seq: u64) {
+        let run = root.join("runs").join(name);
+        fs::create_dir_all(&run).expect("create run");
+        fs::write(
+            run.join("run-meta.json"),
+            format!(
+                r#"{{"status":"success","run_index":{seq},"run_id":"{name}","sync_mode":"snapshot","snapshot_reason":"first_run","started_at_rfc3339_utc":"2026-07-21T00:00:00Z","completed_at_rfc3339_utc":"2026-07-21T00:00:10Z"}}"#
+            ),
+        )
+        .expect("meta");
+        fs::write(
+            run.join("run-summary.json"),
+            format!(
+                r#"{{"runSeq":{seq},"runId":"{name}","startedAtRfc3339Utc":"2026-07-21T00:00:00Z","finishedAtRfc3339Utc":"2026-07-21T00:00:10Z","wallMs":10000,"status":"success","exitCode":0}}"#
+            ),
+        )
+        .expect("summary");
+        fs::write(run.join("result.ccr"), sample_ccr()).expect("ccr");
+    }
+
+    fn write_fake_rpki_client(root: &Path, body: &str) -> PathBuf {
+        let path = root.join("fake-rpki-client.sh");
+        fs::write(&path, body).expect("write fake rpki-client");
+        #[cfg(unix)]
+        {
+            use std::os::unix::fs::PermissionsExt;
+            fs::set_permissions(&path, fs::Permissions::from_mode(0o755)).expect("chmod");
+        }
+        path
+    }
+
+    fn scan_with_ccr_check(
+        root: &Path,
+        previous: Option<&MetricsSnapshot>,
+        bin: &Path,
+        timeout_secs: u64,
+    ) -> MetricsSnapshot {
+        let config = CcrCheckConfig {
+            bin: Some(bin.to_path_buf()),
+            timeout_secs,
+            max_per_scan: 2,
+        };
+        scan_run_root_incremental_with_rtr_dir(root, "test", previous, None, &config).expect("scan")
+    }
+
     fn sample_cir() -> Vec<u8> {
         let rejected = vec![CirRejectedObject {
             object_uri: "rsync://repo.example/a/bad.roa".to_string(),